feat(output): decode hex-wrapped credentials at the persist gate
Gap 1 closed. The output gate finds runs of 32+ hex digits (calibration.HEX_WRAP_MIN_LEN), decodes both byte alignments, keeps text that is at least 80% printable (the base64 bar, now one shared helper in entropy) and runs the credential patterns on it. A hex-wrapped AWS key surfaces as decoded:egress:aws-access-key-id with "in hex run @ N" as evidence; the value is never in it. Credentials only: a hex-wrapped injection is not decoded. Measured before and after on the same trees: all 3583 clean documents (three FP populations and the 78 pinned OKF concepts) are identical line for line under both presets (disposition, labels, severity, count, offset); OKF import stays 47/31. They and 569 READMEs hold 2403 hex runs of 32+ digits, none of which decodes to text. scan_output CPU 122.9 s before, 123.4 s after; one solid hex run at the 1M cap 4.2 s -> 5.8 s. 14 new tests (10 red for the right reason first), 6 of 6 mutations red. Matrix 146/146 + 5/5 gaps, suite 1274 passed, ReDoS sweep 169 patterns.
This commit is contained in:
parent
61d7d521e4
commit
0eab4345bf
14 changed files with 265 additions and 40 deletions
|
|
@ -41,6 +41,14 @@ def test_output_selfsafety_frozen():
|
|||
assert cal.MAX_STYLE_ATTR_CHARS == 2048
|
||||
|
||||
|
||||
def test_hex_wrap_floor_frozen_and_sourced():
|
||||
# Gap 1 (1.10.0): the shortest hex run the output gate decodes and rescans
|
||||
# for credentials. 32 characters is 16 bytes.
|
||||
from llm_ingestion_guard import output
|
||||
assert cal.HEX_WRAP_MIN_LEN == 32
|
||||
assert output.HEX_WRAP_MIN_LEN is cal.HEX_WRAP_MIN_LEN
|
||||
|
||||
|
||||
def test_hidden_markup_thresholds_frozen():
|
||||
assert cal.HIDDEN_FONT_MAX_PX == 4.0
|
||||
assert cal.HIDDEN_COLOUR_TOLERANCE == 8
|
||||
|
|
|
|||
|
|
@ -272,13 +272,13 @@ def test_redos_sweep_floor_and_flag_match_the_published_numbers():
|
|||
assert "2.6 flag threshold" in text
|
||||
|
||||
|
||||
def test_redos_sweep_collector_covers_168_patterns_across_11_tables():
|
||||
# The count docs/LIMITATIONS.md carries as "all 168 compiled patterns across
|
||||
def test_redos_sweep_collector_covers_169_patterns_across_11_tables():
|
||||
# The count docs/LIMITATIONS.md carries as "all 169 compiled patterns across
|
||||
# all eleven regex-bearing modules". A pattern added or removed in `src/`
|
||||
# without re-measuring would drift the doc's claim silently otherwise.
|
||||
assert len(redos_sweep.TABLES) == 11
|
||||
total = sum(len(collect()) for collect in redos_sweep.TABLES.values())
|
||||
assert total == 168
|
||||
assert total == 169
|
||||
|
||||
|
||||
def test_redos_sweep_collector_walks_a_dict_of_patterns():
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ FROZEN_SHA256 = "b1a6d5be5fbfc81263e55135abf11bc6fe349eac82db0d1e84f9e44949b9d22
|
|||
|
||||
# tag -> the `expect` of the caught case that now holds it: a finding label, or
|
||||
# the name of a predicate case when the closing rule adds no label (gap 4).
|
||||
CLOSED: dict[str, str] = {"dedup": "count"}
|
||||
CLOSED: dict[str, str] = {"dedup": "count", "hex-wrapped": "hex-wrapped"}
|
||||
|
||||
|
||||
def _rows() -> list[dict[str, str]]:
|
||||
|
|
|
|||
|
|
@ -152,14 +152,75 @@ def test_base64_wrapped_secret_evidence_never_leaks_the_value():
|
|||
assert AWS_KEY not in (finding.evidence or ""), "decoded evidence leaked the secret"
|
||||
|
||||
|
||||
def test_hex_wrapped_secret_is_a_documented_restgap():
|
||||
# Honest-limit (deliberate boundary, not a silent miss): entropy only exposes
|
||||
# decoded plaintext for base64, not hex, so a hex-wrapped secret is NOT caught.
|
||||
# Documented in README honest-limits; asserted here so the boundary is explicit.
|
||||
def _hex_egress(text: str) -> list:
|
||||
return [f for f in scan_output(text).findings
|
||||
if f.label.startswith("decoded:egress:")]
|
||||
|
||||
|
||||
def test_hex_wrapped_secret_is_caught():
|
||||
# Gap 1 until 1.10.0: a hex run of 32+ characters that decodes to printable
|
||||
# text is fed to scan_secret_egress, as a base64 blob's plaintext already is.
|
||||
hexed = AWS_KEY.encode().hex()
|
||||
report = scan_output("archived reference blob: " + hexed)
|
||||
assert not any(f.label == "decoded:egress:aws-access-key-id"
|
||||
for f in report.findings)
|
||||
labels = {f.label for f in _hex_egress("archived reference blob: " + hexed)}
|
||||
assert "decoded:egress:aws-access-key-id" in labels
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", ["upper", "0x", "odd-tail", "odd-head", "framed"])
|
||||
def test_hex_wrapped_secret_is_caught_in_every_hex_shape(shape):
|
||||
# Case, a 0x prefix and one stray hex digit on either side must not hide the
|
||||
# credential: both byte alignments of the run are decoded.
|
||||
hexed = AWS_KEY.encode().hex()
|
||||
run = {"upper": hexed.upper(), "0x": "0x" + hexed, "odd-tail": hexed + "a",
|
||||
"odd-head": "a" + hexed, "framed": "a" + hexed + "b"}[shape]
|
||||
labels = {f.label for f in _hex_egress("blob " + run + " end")}
|
||||
assert "decoded:egress:aws-access-key-id" in labels, shape
|
||||
|
||||
|
||||
def test_hex_wrapped_secret_survives_a_junk_byte_in_front():
|
||||
# A non-printable byte ahead of the credential is still under the same 80%
|
||||
# printable bar a base64 blob's plaintext meets.
|
||||
hexed = "00" + GITHUB_PAT.encode().hex()
|
||||
labels = {f.label for f in _hex_egress("blob " + hexed)}
|
||||
assert "decoded:egress:github-pat-classic" in labels
|
||||
|
||||
|
||||
def test_hex_wrapped_secret_evidence_never_leaks_the_value_or_its_hex():
|
||||
hexed = AWS_KEY.encode().hex()
|
||||
found = _hex_egress("archived reference blob: " + hexed)
|
||||
assert found, "hex-wrapped AWS key was not surfaced"
|
||||
for finding in found:
|
||||
evidence = finding.evidence or ""
|
||||
assert AWS_KEY not in evidence and hexed not in evidence
|
||||
assert "hex run" in evidence
|
||||
|
||||
|
||||
def test_hex_wrapped_secret_offset_points_at_the_run():
|
||||
prefix = "archived reference blob: "
|
||||
found = _hex_egress(prefix + AWS_KEY.encode().hex())
|
||||
assert [f.offset for f in found] == [len(prefix)]
|
||||
|
||||
|
||||
def test_hex_run_floor_is_32_characters():
|
||||
# 15 bytes -> 30 hex characters: below the floor, not decoded. One byte more
|
||||
# reaches it. The plaintext is a labelled password the egress set reports.
|
||||
below = ("pass='" + "k9Qz7wLp" + "'").encode().hex()
|
||||
at = ("pass='" + "k9Qz7wLpX" + "'").encode().hex()
|
||||
assert len(below) == 30 and len(at) == 32
|
||||
assert _hex_egress("value " + below) == []
|
||||
# Framed to a 32-character run, the credential still spans only 30: the floor
|
||||
# holds for the decoded alignment, not just for the run.
|
||||
assert _hex_egress("value a" + below + "b") == []
|
||||
assert {f.label for f in _hex_egress("value " + at)} == {"decoded:egress:config-password"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", [
|
||||
"sha256 " + "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
|
||||
"commit " + "da39a3ee5e6b4b0d3255bfef95601890afd80709" + " and md5 " + "d41d8cd98f00b204e9800998ecf8427e",
|
||||
"plain " + b"an ordinary sentence with no credential in it".hex(),
|
||||
"binary " + bytes(range(0, 256, 3)).hex(),
|
||||
])
|
||||
def test_hex_runs_without_a_credential_give_no_decoded_finding(text):
|
||||
assert _hex_egress(text) == []
|
||||
|
||||
|
||||
def test_aggregates_lexicon_and_egress_findings():
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue