248 lines
13 KiB
JSON
248 lines
13 KiB
JSON
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": "https://git.fromaitochitta.com/open/llm-security-commons/raw/branch/main/schema/conformance-declaration.schema.json",
|
|
"version": "0.1.0",
|
|
"status": "normative",
|
|
"title": "Conformance declaration",
|
|
"description": "The shape a runtime publishes alongside a conformance result, satisfying the MUST in spec/conformance-corpus.md section 1.1: declare the set of commons data files you implement, and publish that set with your result. Normative for the field names, types and the two arithmetic invariants stated in `invariants`. A result published without a declaration in this shape is not checkable by anyone but its author, which is most of the reason section 1.1 exists.",
|
|
"$comment": "Authored in this repository rather than extracted from a runtime, because no runtime had one to extract - section 1.1 shipped in commons v0.2.0 and created the obligation. Requested by llm-security (coord message 2026-08-11T10:46:28Z) in those terms: 'runtime name, the commit measured, and the list of commons file paths implemented', with the stated reason that two runtimes publishing free-form declarations makes `83/83 + 6 not-applicable` unparseable by anyone but its author. Deliberately NOT a gate: commons runs nothing (see CLAUDE.md charter), and llm-security explicitly did not ask for validation. Publishing the shape is the whole contribution.",
|
|
"requested_by": {
|
|
"runtime": "llm-security",
|
|
"coord_message_date": "2026-08-11T10:46:28Z",
|
|
"asked_for": "runtime name, the commit measured, the list of implemented commons file paths",
|
|
"beyond_the_ask": "`declaration_source`, `error`, and the enumeration arrays are not in their request. Each exists to make a section 1.1 MUST checkable rather than merely stated - see `invariants` and the per-property notes. Nothing here is required in order to satisfy their stated need."
|
|
},
|
|
"invariants": {
|
|
"$comment": "Two constraints that JSON Schema cannot express, stated normatively here because they are the point of the format rather than decoration on it. A validator that checks the schema and not these has checked the shape of a claim without checking the claim.",
|
|
"denominator_intact": "`result.total` MUST equal `passed + failed + not_applicable + error`. This is section 1.1's fence in arithmetic form: `76/83 passed` and `76 passed, 7 not-applicable` describe different runtimes, and only the second one can be checked. A declaration whose parts do not sum to its total has dropped cases somewhere between the run and the publication.",
|
|
"total_matches_corpus": "`result.total` MUST equal the number of cases the corpus contains at `commons_commit` - conformance/manifest.json's case count at that commit, not at HEAD. A runtime that ran fewer cases than the corpus holds has skipped some, which section 1 forbids outright."
|
|
},
|
|
"type": "object",
|
|
"$ref": "#/$defs/declaration",
|
|
"$defs": {
|
|
"declaration": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": [
|
|
"runtime",
|
|
"commons_commit",
|
|
"declared_tables",
|
|
"result"
|
|
],
|
|
"properties": {
|
|
"runtime": {
|
|
"type": "string",
|
|
"minLength": 1,
|
|
"description": "The runtime publishing this result, named the same way it is named in the lexicon's `aliases` keys where it appears there (for example `llm_security`, `llm_ingestion_guard`). A runtime not registered in any commons alias block names itself freely; the constraint exists so a reader can join a declaration to an alias, not to restrict who may declare."
|
|
},
|
|
"runtime_commit": {
|
|
"type": "string",
|
|
"minLength": 7,
|
|
"description": "The runtime's own commit that produced this result. SHOULD be present: without it the result cannot be reproduced, only believed."
|
|
},
|
|
"commons_commit": {
|
|
"type": "string",
|
|
"minLength": 7,
|
|
"description": "The commons commit whose corpus was measured. REQUIRED, and it is the coordinate the whole declaration hangs on - case counts, case ids and table paths all move with it. A declaration citing a tag instead SHOULD give the commit the tag resolved to, because a tag can be moved and a commit cannot."
|
|
},
|
|
"commons_version": {
|
|
"type": "string",
|
|
"description": "The commons release tag measured, if the measurement was taken at a tag (for example `v0.3.0`). Informative convenience; `commons_commit` is the authority."
|
|
},
|
|
"declared_tables": {
|
|
"type": "array",
|
|
"minItems": 1,
|
|
"uniqueItems": true,
|
|
"items": {
|
|
"type": "string",
|
|
"pattern": "^[a-z0-9][a-z0-9._/-]*\\.json$"
|
|
},
|
|
"description": "The set of commons data files this runtime implements, as repository-relative paths exactly as they appear in the commons tree at `commons_commit` (for example `lexicon/injection-lexicon.json`). This is the declaration section 1.1 requires. A case whose `scope` names a file outside this set is `not-applicable`; a case whose scope names a file inside it MUST be run, including the ones it fails."
|
|
},
|
|
"declaration_source": {
|
|
"enum": [
|
|
"derived-from-runner",
|
|
"hand-maintained"
|
|
],
|
|
"description": "Whether `declared_tables` is READ FROM the same constant the runner uses to accept or reject a scope (`derived-from-runner`), or maintained as a separate list beside it (`hand-maintained`). SHOULD be `derived-from-runner`. The distinction is not cosmetic: section 1.1 forbids narrowing a declared set in order to convert failures into `not-applicable` results, and only a derived declaration makes that fence structural - narrowing it is then a code change with tests behind it rather than an edit to a file nobody reads. The field exists so a reader can tell which kind of claim they are reading. Formulation owed to llm-security, who stated the constraint for their own side before commons had a field for it: 'a hand-maintained declaration is a claim; a derived one is a measurement.'"
|
|
},
|
|
"measured_date": {
|
|
"type": "string",
|
|
"format": "date",
|
|
"description": "The date the result was produced, ISO 8601. SHOULD be present: a declaration with no date cannot be ordered against a later one."
|
|
},
|
|
"result": {
|
|
"$ref": "#/$defs/result"
|
|
},
|
|
"not_applicable_cases": {
|
|
"$ref": "#/$defs/caseIdList",
|
|
"description": "Every case reported `not-applicable`, by case id. Section 1.1 requires that such cases still be enumerated in the result; this array is that enumeration, and it is REQUIRED whenever the count is non-zero. A bare count would satisfy the arithmetic and defeat the purpose - the reader needs to see WHICH questions were never addressed to this runtime in order to judge whether the declared set explains them."
|
|
},
|
|
"failed_cases": {
|
|
"$ref": "#/$defs/caseIdList",
|
|
"description": "Every case reported `fail`, by case id. REQUIRED whenever the count is non-zero. A failure this runtime is not naming is a failure nobody can reproduce."
|
|
},
|
|
"error_cases": {
|
|
"$ref": "#/$defs/caseIdList",
|
|
"description": "Every case reported as an error under section 1 - the runtime tried to run it and could not - by case id. REQUIRED whenever the count is non-zero. Kept distinct from `not_applicable_cases` because section 1.1 turns on exactly that distinction: an error means *this runtime tried and could not*, `not-applicable` means *this question was never addressed to this runtime*. Collapsing the two reports a capability gap as an architectural choice, or the reverse."
|
|
},
|
|
"notes": {
|
|
"type": "string",
|
|
"description": "Free text for anything the fields above cannot carry. MUST NOT be used to qualify a count: a number that needs prose to be read correctly is the wrong number."
|
|
}
|
|
},
|
|
"allOf": [
|
|
{
|
|
"if": {
|
|
"properties": {
|
|
"result": {
|
|
"properties": {
|
|
"not_applicable": {
|
|
"minimum": 1
|
|
}
|
|
},
|
|
"required": [
|
|
"not_applicable"
|
|
]
|
|
}
|
|
},
|
|
"required": [
|
|
"result"
|
|
]
|
|
},
|
|
"then": {
|
|
"required": [
|
|
"not_applicable_cases"
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"if": {
|
|
"properties": {
|
|
"result": {
|
|
"properties": {
|
|
"failed": {
|
|
"minimum": 1
|
|
}
|
|
},
|
|
"required": [
|
|
"failed"
|
|
]
|
|
}
|
|
},
|
|
"required": [
|
|
"result"
|
|
]
|
|
},
|
|
"then": {
|
|
"required": [
|
|
"failed_cases"
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"if": {
|
|
"properties": {
|
|
"result": {
|
|
"properties": {
|
|
"error": {
|
|
"minimum": 1
|
|
}
|
|
},
|
|
"required": [
|
|
"error"
|
|
]
|
|
}
|
|
},
|
|
"required": [
|
|
"result"
|
|
]
|
|
},
|
|
"then": {
|
|
"required": [
|
|
"error_cases"
|
|
]
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"result": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": [
|
|
"total",
|
|
"passed",
|
|
"failed",
|
|
"not_applicable",
|
|
"error"
|
|
],
|
|
"$comment": "All five counts are REQUIRED, including the ones that are zero. An absent count is indistinguishable from a count the runtime did not track, and the `denominator_intact` invariant cannot be checked against a declaration with holes in it. `0` is information; a missing key is not.",
|
|
"properties": {
|
|
"total": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "Cases enumerated. MUST equal the corpus case count at `commons_commit`, and MUST equal the sum of the four verdict counts. See `invariants`."
|
|
},
|
|
"passed": {
|
|
"type": "integer",
|
|
"minimum": 0
|
|
},
|
|
"failed": {
|
|
"type": "integer",
|
|
"minimum": 0
|
|
},
|
|
"not_applicable": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "Cases whose `scope` names a data file outside `declared_tables`. MUST NOT be counted as passes and MUST NOT be dropped from `total`."
|
|
},
|
|
"error": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "Cases the runtime tried to run and could not, per section 1. Distinct from `not_applicable`."
|
|
},
|
|
"match": {
|
|
"type": "string",
|
|
"description": "The match semantics the run used, if the corpus offers more than one (today: `exact-within-scope`). Informative; the per-case `match` field in each expected.json is the authority."
|
|
}
|
|
}
|
|
},
|
|
"caseIdList": {
|
|
"type": "array",
|
|
"uniqueItems": true,
|
|
"items": {
|
|
"type": "string",
|
|
"pattern": "^[a-z0-9][a-z0-9-]*__[a-z0-9-]+$"
|
|
},
|
|
"$comment": "Case ids as they appear on disk under conformance/, which per conformance/manifest.json are derived from a pattern id by replacing ':' with '__'. The pattern here enforces that shape rather than the derivation itself, since the derivation is the manifest's contract and not this schema's."
|
|
}
|
|
},
|
|
"example_$comment": "Illustrative only, not a record of any measurement. The counts are the shape llm-security stated it will publish once it pulls a commons carrying section 1.1 - see requested_by - and are NOT a result this repository has been shown. This note sits OUTSIDE `example` on purpose: `declaration` closes additionalProperties, so an example carrying its own $comment would not validate against the schema it illustrates.",
|
|
"example": {
|
|
"runtime": "llm_security",
|
|
"runtime_commit": "b1ba1fb",
|
|
"commons_commit": "0000000",
|
|
"commons_version": "v0.3.0",
|
|
"declared_tables": [
|
|
"lexicon/injection-lexicon.json"
|
|
],
|
|
"declaration_source": "derived-from-runner",
|
|
"measured_date": "2026-08-11",
|
|
"result": {
|
|
"total": 89,
|
|
"passed": 83,
|
|
"failed": 0,
|
|
"not_applicable": 6,
|
|
"error": 0,
|
|
"match": "exact-within-scope"
|
|
},
|
|
"not_applicable_cases": [
|
|
"active__autolink",
|
|
"active__data-uri",
|
|
"active__markdown-image",
|
|
"active__markdown-link",
|
|
"active__raw-html",
|
|
"active__reference-link"
|
|
]
|
|
}
|
|
}
|