1
0
Fork 0

feat(output): decode hex-wrapped credentials at the persist gate

Gap 1 closed. The output gate finds runs of 32+ hex digits
(calibration.HEX_WRAP_MIN_LEN), decodes both byte alignments, keeps text
that is at least 80% printable (the base64 bar, now one shared helper in
entropy) and runs the credential patterns on it. A hex-wrapped AWS key
surfaces as decoded:egress:aws-access-key-id with "in hex run @ N" as
evidence; the value is never in it. Credentials only: a hex-wrapped
injection is not decoded.

Measured before and after on the same trees: all 3583 clean documents
(three FP populations and the 78 pinned OKF concepts) are identical line
for line under both presets (disposition, labels, severity, count,
offset); OKF import stays 47/31. They and 569 READMEs hold 2403 hex runs
of 32+ digits, none of which decodes to text. scan_output CPU 122.9 s
before, 123.4 s after; one solid hex run at the 1M cap 4.2 s -> 5.8 s.

14 new tests (10 red for the right reason first), 6 of 6 mutations red.
Matrix 146/146 + 5/5 gaps, suite 1274 passed, ReDoS sweep 169 patterns.
This commit is contained in:
Kjell Tore Guttormsen 2026-11-01 17:06:44 +01:00
commit 0eab4345bf
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
14 changed files with 265 additions and 40 deletions

View file

@ -41,6 +41,14 @@ def test_output_selfsafety_frozen():
assert cal.MAX_STYLE_ATTR_CHARS == 2048
def test_hex_wrap_floor_frozen_and_sourced():
# Gap 1 (1.10.0): the shortest hex run the output gate decodes and rescans
# for credentials. 32 characters is 16 bytes.
from llm_ingestion_guard import output
assert cal.HEX_WRAP_MIN_LEN == 32
assert output.HEX_WRAP_MIN_LEN is cal.HEX_WRAP_MIN_LEN
def test_hidden_markup_thresholds_frozen():
assert cal.HIDDEN_FONT_MAX_PX == 4.0
assert cal.HIDDEN_COLOUR_TOLERANCE == 8

View file

@ -272,13 +272,13 @@ def test_redos_sweep_floor_and_flag_match_the_published_numbers():
assert "2.6 flag threshold" in text
def test_redos_sweep_collector_covers_168_patterns_across_11_tables():
# The count docs/LIMITATIONS.md carries as "all 168 compiled patterns across
def test_redos_sweep_collector_covers_169_patterns_across_11_tables():
# The count docs/LIMITATIONS.md carries as "all 169 compiled patterns across
# all eleven regex-bearing modules". A pattern added or removed in `src/`
# without re-measuring would drift the doc's claim silently otherwise.
assert len(redos_sweep.TABLES) == 11
total = sum(len(collect()) for collect in redos_sweep.TABLES.values())
assert total == 168
assert total == 169
def test_redos_sweep_collector_walks_a_dict_of_patterns():

View file

@ -24,7 +24,7 @@ FROZEN_SHA256 = "b1a6d5be5fbfc81263e55135abf11bc6fe349eac82db0d1e84f9e44949b9d22
# tag -> the `expect` of the caught case that now holds it: a finding label, or
# the name of a predicate case when the closing rule adds no label (gap 4).
CLOSED: dict[str, str] = {"dedup": "count"}
CLOSED: dict[str, str] = {"dedup": "count", "hex-wrapped": "hex-wrapped"}
def _rows() -> list[dict[str, str]]:

View file

@ -152,14 +152,75 @@ def test_base64_wrapped_secret_evidence_never_leaks_the_value():
assert AWS_KEY not in (finding.evidence or ""), "decoded evidence leaked the secret"
def test_hex_wrapped_secret_is_a_documented_restgap():
# Honest-limit (deliberate boundary, not a silent miss): entropy only exposes
# decoded plaintext for base64, not hex, so a hex-wrapped secret is NOT caught.
# Documented in README honest-limits; asserted here so the boundary is explicit.
def _hex_egress(text: str) -> list:
return [f for f in scan_output(text).findings
if f.label.startswith("decoded:egress:")]
def test_hex_wrapped_secret_is_caught():
# Gap 1 until 1.10.0: a hex run of 32+ characters that decodes to printable
# text is fed to scan_secret_egress, as a base64 blob's plaintext already is.
hexed = AWS_KEY.encode().hex()
report = scan_output("archived reference blob: " + hexed)
assert not any(f.label == "decoded:egress:aws-access-key-id"
for f in report.findings)
labels = {f.label for f in _hex_egress("archived reference blob: " + hexed)}
assert "decoded:egress:aws-access-key-id" in labels
@pytest.mark.parametrize("shape", ["upper", "0x", "odd-tail", "odd-head", "framed"])
def test_hex_wrapped_secret_is_caught_in_every_hex_shape(shape):
# Case, a 0x prefix and one stray hex digit on either side must not hide the
# credential: both byte alignments of the run are decoded.
hexed = AWS_KEY.encode().hex()
run = {"upper": hexed.upper(), "0x": "0x" + hexed, "odd-tail": hexed + "a",
"odd-head": "a" + hexed, "framed": "a" + hexed + "b"}[shape]
labels = {f.label for f in _hex_egress("blob " + run + " end")}
assert "decoded:egress:aws-access-key-id" in labels, shape
def test_hex_wrapped_secret_survives_a_junk_byte_in_front():
# A non-printable byte ahead of the credential is still under the same 80%
# printable bar a base64 blob's plaintext meets.
hexed = "00" + GITHUB_PAT.encode().hex()
labels = {f.label for f in _hex_egress("blob " + hexed)}
assert "decoded:egress:github-pat-classic" in labels
def test_hex_wrapped_secret_evidence_never_leaks_the_value_or_its_hex():
hexed = AWS_KEY.encode().hex()
found = _hex_egress("archived reference blob: " + hexed)
assert found, "hex-wrapped AWS key was not surfaced"
for finding in found:
evidence = finding.evidence or ""
assert AWS_KEY not in evidence and hexed not in evidence
assert "hex run" in evidence
def test_hex_wrapped_secret_offset_points_at_the_run():
prefix = "archived reference blob: "
found = _hex_egress(prefix + AWS_KEY.encode().hex())
assert [f.offset for f in found] == [len(prefix)]
def test_hex_run_floor_is_32_characters():
# 15 bytes -> 30 hex characters: below the floor, not decoded. One byte more
# reaches it. The plaintext is a labelled password the egress set reports.
below = ("pass='" + "k9Qz7wLp" + "'").encode().hex()
at = ("pass='" + "k9Qz7wLpX" + "'").encode().hex()
assert len(below) == 30 and len(at) == 32
assert _hex_egress("value " + below) == []
# Framed to a 32-character run, the credential still spans only 30: the floor
# holds for the decoded alignment, not just for the run.
assert _hex_egress("value a" + below + "b") == []
assert {f.label for f in _hex_egress("value " + at)} == {"decoded:egress:config-password"}
@pytest.mark.parametrize("text", [
"sha256 " + "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
"commit " + "da39a3ee5e6b4b0d3255bfef95601890afd80709" + " and md5 " + "d41d8cd98f00b204e9800998ecf8427e",
"plain " + b"an ordinary sentence with no credential in it".hex(),
"binary " + bytes(range(0, 256, 3)).hex(),
])
def test_hex_runs_without_a_credential_give_no_decoded_finding(text):
assert _hex_egress(text) == []
def test_aggregates_lexicon_and_egress_findings():