154 lines
7.7 KiB
JSON
154 lines
7.7 KiB
JSON
{
|
|
"version": "0.1.0",
|
|
"id": "llm-security-commons/conformance",
|
|
"description": "Enumeration and measurement header for the conformance corpus. Every case directory holds input.txt (the exact bytes to scan) and expected.json (the findings a conforming runtime must produce). The normative reading of those files is spec/conformance-corpus.md; this file records where the cases came from and what was measured.",
|
|
"$comment": "Fixture files carry no individual version field. The corpus is versioned as a whole, here — a case is added, removed or corrected by bumping this version, and a case-id change is a MAJOR bump because consumers name cases.",
|
|
"case_id_derivation": {
|
|
"rule": "case_id = pattern_id with ':' replaced by '__'",
|
|
"reverse": "pattern_id = case_id with '__' replaced by ':'",
|
|
"why": "':' is not a legal filename character on Windows, and this repository is fork-and-own. '__' does not occur in the ratified id space, so the transform is one-to-one — verified collision-free across all 83.",
|
|
"stability": "A case id is a stable identifier. Changing one is a BREAKING change."
|
|
},
|
|
"match_semantics": "exact-within-scope",
|
|
"scope_covered": [
|
|
"lexicon/injection-lexicon.json"
|
|
],
|
|
"scope_planned": {
|
|
"$comment": "Named rather than faked. The guard's coverage matrix holds 11 further convertible cases (3 carrier, 7 active-content, 1 secret-egress), but the lexicon is the only table whose finding id space is ratified by both seeding runtimes. Fixtures for the others would mint a cross-runtime contract this repository has no agreement for, so they wait for one.",
|
|
"codepoints/carriers.json": 3,
|
|
"signatures/active-content.json": 7,
|
|
"signatures/secret-egress.json": 1
|
|
},
|
|
"payload_provenance": {
|
|
"source_repo": "llm-ingestion-pipeline-security",
|
|
"source_file": "src/llm_ingestion_guard/coverage.py",
|
|
"source_export": "_LEX_PAYLOADS",
|
|
"source_commit": "0bf07295c2191d5061537834abf22929f7d50826",
|
|
"source_version": "0.3.4",
|
|
"$comment": "The inputs were authored by one of the two runtimes, as one payload per pattern id, and are reproduced verbatim. That asymmetry is stated rather than averaged away: what makes them usable as a cross-runtime corpus is not their origin but the measurement below, which ran them through the other runtime as well and found the same lexicon verdict on every one.",
|
|
"id_set_check": "The 83 payload keys and the 83 commons pattern ids are the same set — compared, not assumed."
|
|
},
|
|
"measurement": {
|
|
"date": "2026-08-10",
|
|
"method": "Each payload was run through both runtimes' PUBLIC entry point — not through a rebuilt regex table — and the resulting finding labels were mapped to commons pattern ids through the lexicon's own aliases block. Comparing at the entry point is deliberate: a table-level comparison produces a number that describes neither runtime.",
|
|
"runtimes": [
|
|
{
|
|
"name": "llm_security",
|
|
"repo": "ssh://git@git.fromaitochitta.com/open/llm-security.git",
|
|
"commit": "b0de0ca6d86ce697f39669d177c2c2654c280128",
|
|
"entry_point": "scanForInjection(text) — scanners/lib/injection-patterns.mjs",
|
|
"engine": "Node v25.8.2",
|
|
"covers": "normalisation, homoglyph folding, the rot13 variant and all four pattern arrays",
|
|
"measurement_limit": "This entry point is the injection scanner. Whether this runtime raises findings from other commons tables on these inputs was NOT measured, so observed_out_of_scope carries no entry for it — absence of a key means unmeasured, not measured-empty."
|
|
},
|
|
{
|
|
"name": "llm_ingestion_guard",
|
|
"repo": "llm-ingestion-pipeline-security",
|
|
"commit": "0bf07295c2191d5061537834abf22929f7d50826",
|
|
"version": "0.3.4",
|
|
"entry_point": "scan_output(text, source=Source.OUTPUT)",
|
|
"engine": "CPython 3.14.0",
|
|
"covers": "the whole output gate, which is more than the lexicon",
|
|
"measurement_limit": "Non-lexicon findings this gate raised are recorded per case in observed_out_of_scope as informative evidence, never as expectation."
|
|
}
|
|
],
|
|
"results": {
|
|
"cases": 83,
|
|
"asserted_id_present_in_both_runtimes": 83,
|
|
"lexicon_id_sets_identical_between_runtimes": 83,
|
|
"cases_with_non_lexicon_residue_in_the_guard": 7
|
|
},
|
|
"known_divergence": {
|
|
"$comment": "13 of the 83 patterns are recorded in docs/lexicon-port-divergence.md as behaviourally divergent between the two ports. That divergence is real and unresolved, and it is NOT visible here: it was measured on witness inputs — an attribute run padded past 256 characters, an interior '<', an unclosed <script> — and none of those shapes occurs in these payloads. All 13 produce identical lexicon id sets on their own case input, so their fixtures carry no marker and no caveat. A caveat would assert a doubt the measurement disproves for these inputs.",
|
|
"patterns_covered": 13,
|
|
"patterns_agreeing_on_their_case_input": 13,
|
|
"see": "docs/lexicon-port-divergence.md"
|
|
}
|
|
},
|
|
"count": 83,
|
|
"cases": [
|
|
"override__ignore-previous",
|
|
"override__forget-instructions",
|
|
"override__disregard",
|
|
"override__override-instructions",
|
|
"override__suspend-constraints",
|
|
"override__prior-constraints",
|
|
"spoofed-header__hash-system",
|
|
"spoofed-header__inst",
|
|
"spoofed-header__pipe-system",
|
|
"spoofed-tag__system-open",
|
|
"spoofed-tag__system-close",
|
|
"spoofed-header__instructions",
|
|
"identity__no-longer",
|
|
"identity__you-are-now",
|
|
"identity__new-role",
|
|
"identity__new-directive",
|
|
"identity__pretend",
|
|
"config__ignore-claudemd",
|
|
"config__override-system-prompt",
|
|
"config__bypass-safety",
|
|
"config__disable-filtering",
|
|
"normalization__prioritize-user",
|
|
"normalization__user-precedence",
|
|
"normalization__claudemd-conflict",
|
|
"normalization__skip-confirmation",
|
|
"reframing__actual-task",
|
|
"reframing__real-purpose",
|
|
"reframing__should-actually-do",
|
|
"hidden-comment__agent-directed",
|
|
"html-obfuscation__display-none",
|
|
"html-obfuscation__visibility-hidden",
|
|
"html-obfuscation__offscreen",
|
|
"html-obfuscation__zero-font",
|
|
"html-obfuscation__zero-opacity",
|
|
"html-obfuscation__zero-size-overflow",
|
|
"html-obfuscation__aria-label",
|
|
"evasion__educational",
|
|
"evasion__hypothetical",
|
|
"evasion__fictional",
|
|
"evasion__red-team",
|
|
"evasion__security-audit-sim",
|
|
"evasion__test-pretense",
|
|
"evasion__thought-experiment",
|
|
"evasion__research-purpose",
|
|
"evasion__academic-context",
|
|
"hitl-trap__approve-immediately",
|
|
"hitl-trap__no-need-review",
|
|
"hitl-trap__just-approve",
|
|
"hitl-trap__suppress-review",
|
|
"hitl-trap__brevity-skip",
|
|
"hitl-trap__too-long",
|
|
"hitl-trap__details-omitted",
|
|
"hitl-trap__scope-minimization",
|
|
"leetspeak__ignore-previous",
|
|
"leetspeak__forget-instructions",
|
|
"leetspeak__disregard",
|
|
"leetspeak__override",
|
|
"homoglyph__cyrillic-latin-mix",
|
|
"unicode__zero-width-in-word",
|
|
"indirect__instruction-to-ai",
|
|
"indirect__direct-address",
|
|
"indirect__trigger-based",
|
|
"multi-lang__french",
|
|
"multi-lang__spanish",
|
|
"multi-lang__german",
|
|
"markdown__link-ref-comment",
|
|
"data-uri__executable",
|
|
"markdown__link-anchor-injection",
|
|
"sub-agent__spawn-dangerous",
|
|
"sub-agent__delegate-bypass",
|
|
"nl-indirection__fetch-execute",
|
|
"nl-indirection__send-external",
|
|
"nl-indirection__read-dotfile",
|
|
"nl-indirection__extract-exfiltrate",
|
|
"nl-indirection__download-execute",
|
|
"hybrid-p2sql__inject-sql",
|
|
"hybrid-p2sql__sql-inject",
|
|
"hybrid-recursive__inject-output",
|
|
"hybrid-recursive__conditional-response",
|
|
"hybrid-xss__script-tag",
|
|
"hybrid-xss__javascript-uri",
|
|
"hybrid-xss__event-handler",
|
|
"hybrid-xss__iframe-src"
|
|
]
|
|
}
|