{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://git.fromaitochitta.com/open/llm-security-commons/raw/branch/main/schema/conformance-declaration.schema.json",
  "version": "0.1.0",
  "status": "normative",
  "title": "Conformance declaration",
  "description": "The shape a runtime publishes alongside a conformance result, satisfying the MUST in spec/conformance-corpus.md section 1.1: declare the set of commons data files you implement, and publish that set with your result. Normative for the field names, types and the two arithmetic invariants stated in `invariants`. A result published without a declaration in this shape is not checkable by anyone but its author, which is most of the reason section 1.1 exists.",
  "$comment": "Authored in this repository rather than extracted from a runtime, because no runtime had one to extract - section 1.1 shipped in commons v0.2.0 and created the obligation. Requested by llm-security (coord message 2026-08-11T10:46:28Z) in those terms: 'runtime name, the commit measured, and the list of commons file paths implemented', with the stated reason that two runtimes publishing free-form declarations makes `83/83 + 6 not-applicable` unparseable by anyone but its author. Deliberately NOT a gate: commons runs nothing (see CLAUDE.md charter), and llm-security explicitly did not ask for validation. Publishing the shape is the whole contribution.",
  "requested_by": {
    "runtime": "llm-security",
    "coord_message_date": "2026-08-11T10:46:28Z",
    "asked_for": "runtime name, the commit measured, the list of implemented commons file paths",
    "beyond_the_ask": "`declaration_source`, `error`, and the enumeration arrays are not in their request. Each exists to make a section 1.1 MUST checkable rather than merely stated - see `invariants` and the per-property notes. Nothing here is required in order to satisfy their stated need."
  },
  "invariants": {
    "$comment": "Two constraints that JSON Schema cannot express, stated normatively here because they are the point of the format rather than decoration on it. A validator that checks the schema and not these has checked the shape of a claim without checking the claim.",
    "denominator_intact": "`result.total` MUST equal `passed + failed + not_applicable + error`. This is section 1.1's fence in arithmetic form: `76/83 passed` and `76 passed, 7 not-applicable` describe different runtimes, and only the second one can be checked. A declaration whose parts do not sum to its total has dropped cases somewhere between the run and the publication.",
    "total_matches_corpus": "`result.total` MUST equal the number of cases the corpus contains at `commons_commit` - conformance/manifest.json's case count at that commit, not at HEAD. A runtime that ran fewer cases than the corpus holds has skipped some, which section 1 forbids outright."
  },
  "type": "object",
  "$ref": "#/$defs/declaration",
  "$defs": {
    "declaration": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "runtime",
        "commons_commit",
        "declared_tables",
        "result"
      ],
      "properties": {
        "runtime": {
          "type": "string",
          "minLength": 1,
          "description": "The runtime publishing this result, named the same way it is named in the lexicon's `aliases` keys where it appears there (for example `llm_security`, `llm_ingestion_guard`). A runtime not registered in any commons alias block names itself freely; the constraint exists so a reader can join a declaration to an alias, not to restrict who may declare."
        },
        "runtime_commit": {
          "type": "string",
          "minLength": 7,
          "description": "The runtime's own commit that produced this result. SHOULD be present: without it the result cannot be reproduced, only believed."
        },
        "commons_commit": {
          "type": "string",
          "minLength": 7,
          "description": "The commons commit whose corpus was measured. REQUIRED, and it is the coordinate the whole declaration hangs on - case counts, case ids and table paths all move with it. A declaration citing a tag instead SHOULD give the commit the tag resolved to, because a tag can be moved and a commit cannot."
        },
        "commons_version": {
          "type": "string",
          "description": "The commons release tag measured, if the measurement was taken at a tag (for example `v0.3.0`). Informative convenience; `commons_commit` is the authority."
        },
        "declared_tables": {
          "type": "array",
          "minItems": 1,
          "uniqueItems": true,
          "items": {
            "type": "string",
            "pattern": "^[a-z0-9][a-z0-9._/-]*\\.json$"
          },
          "description": "The set of commons data files this runtime implements, as repository-relative paths exactly as they appear in the commons tree at `commons_commit` (for example `lexicon/injection-lexicon.json`). This is the declaration section 1.1 requires. A case whose `scope` names a file outside this set is `not-applicable`; a case whose scope names a file inside it MUST be run, including the ones it fails."
        },
        "declaration_source": {
          "enum": [
            "derived-from-runner",
            "hand-maintained"
          ],
          "description": "Whether `declared_tables` is READ FROM the same constant the runner uses to accept or reject a scope (`derived-from-runner`), or maintained as a separate list beside it (`hand-maintained`). SHOULD be `derived-from-runner`. The distinction is not cosmetic: section 1.1 forbids narrowing a declared set in order to convert failures into `not-applicable` results, and only a derived declaration makes that fence structural - narrowing it is then a code change with tests behind it rather than an edit to a file nobody reads. The field exists so a reader can tell which kind of claim they are reading. Formulation owed to llm-security, who stated the constraint for their own side before commons had a field for it: 'a hand-maintained declaration is a claim; a derived one is a measurement.'"
        },
        "measured_date": {
          "type": "string",
          "format": "date",
          "description": "The date the result was produced, ISO 8601. SHOULD be present: a declaration with no date cannot be ordered against a later one."
        },
        "result": {
          "$ref": "#/$defs/result"
        },
        "not_applicable_cases": {
          "$ref": "#/$defs/caseIdList",
          "description": "Every case reported `not-applicable`, by case id. Section 1.1 requires that such cases still be enumerated in the result; this array is that enumeration, and it is REQUIRED whenever the count is non-zero. A bare count would satisfy the arithmetic and defeat the purpose - the reader needs to see WHICH questions were never addressed to this runtime in order to judge whether the declared set explains them."
        },
        "failed_cases": {
          "$ref": "#/$defs/caseIdList",
          "description": "Every case reported `fail`, by case id. REQUIRED whenever the count is non-zero. A failure this runtime is not naming is a failure nobody can reproduce."
        },
        "error_cases": {
          "$ref": "#/$defs/caseIdList",
          "description": "Every case reported as an error under section 1 - the runtime tried to run it and could not - by case id. REQUIRED whenever the count is non-zero. Kept distinct from `not_applicable_cases` because section 1.1 turns on exactly that distinction: an error means *this runtime tried and could not*, `not-applicable` means *this question was never addressed to this runtime*. Collapsing the two reports a capability gap as an architectural choice, or the reverse."
        },
        "notes": {
          "type": "string",
          "description": "Free text for anything the fields above cannot carry. MUST NOT be used to qualify a count: a number that needs prose to be read correctly is the wrong number."
        }
      },
      "allOf": [
        {
          "if": {
            "properties": {
              "result": {
                "properties": {
                  "not_applicable": {
                    "minimum": 1
                  }
                },
                "required": [
                  "not_applicable"
                ]
              }
            },
            "required": [
              "result"
            ]
          },
          "then": {
            "required": [
              "not_applicable_cases"
            ]
          }
        },
        {
          "if": {
            "properties": {
              "result": {
                "properties": {
                  "failed": {
                    "minimum": 1
                  }
                },
                "required": [
                  "failed"
                ]
              }
            },
            "required": [
              "result"
            ]
          },
          "then": {
            "required": [
              "failed_cases"
            ]
          }
        },
        {
          "if": {
            "properties": {
              "result": {
                "properties": {
                  "error": {
                    "minimum": 1
                  }
                },
                "required": [
                  "error"
                ]
              }
            },
            "required": [
              "result"
            ]
          },
          "then": {
            "required": [
              "error_cases"
            ]
          }
        }
      ]
    },
    "result": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "total",
        "passed",
        "failed",
        "not_applicable",
        "error"
      ],
      "$comment": "All five counts are REQUIRED, including the ones that are zero. An absent count is indistinguishable from a count the runtime did not track, and the `denominator_intact` invariant cannot be checked against a declaration with holes in it. `0` is information; a missing key is not.",
      "properties": {
        "total": {
          "type": "integer",
          "minimum": 0,
          "description": "Cases enumerated. MUST equal the corpus case count at `commons_commit`, and MUST equal the sum of the four verdict counts. See `invariants`."
        },
        "passed": {
          "type": "integer",
          "minimum": 0
        },
        "failed": {
          "type": "integer",
          "minimum": 0
        },
        "not_applicable": {
          "type": "integer",
          "minimum": 0,
          "description": "Cases whose `scope` names a data file outside `declared_tables`. MUST NOT be counted as passes and MUST NOT be dropped from `total`."
        },
        "error": {
          "type": "integer",
          "minimum": 0,
          "description": "Cases the runtime tried to run and could not, per section 1. Distinct from `not_applicable`."
        },
        "match": {
          "type": "string",
          "description": "The match semantics the run used, if the corpus offers more than one (today: `exact-within-scope`). Informative; the per-case `match` field in each expected.json is the authority."
        }
      }
    },
    "caseIdList": {
      "type": "array",
      "uniqueItems": true,
      "items": {
        "type": "string",
        "pattern": "^[a-z0-9][a-z0-9-]*__[a-z0-9-]+$"
      },
      "$comment": "Case ids as they appear on disk under conformance/, which per conformance/manifest.json are derived from a pattern id by replacing ':' with '__'. The pattern here enforces that shape rather than the derivation itself, since the derivation is the manifest's contract and not this schema's."
    }
  },
  "example_$comment": "Illustrative only, not a record of any measurement. The counts are the shape llm-security stated it will publish once it pulls a commons carrying section 1.1 - see requested_by - and are NOT a result this repository has been shown. This note sits OUTSIDE `example` on purpose: `declaration` closes additionalProperties, so an example carrying its own $comment would not validate against the schema it illustrates.",
  "example": {
    "runtime": "llm_security",
    "runtime_commit": "b1ba1fb",
    "commons_commit": "0000000",
    "commons_version": "v0.3.0",
    "declared_tables": [
      "lexicon/injection-lexicon.json"
    ],
    "declaration_source": "derived-from-runner",
    "measured_date": "2026-08-11",
    "result": {
      "total": 89,
      "passed": 83,
      "failed": 0,
      "not_applicable": 6,
      "error": 0,
      "match": "exact-within-scope"
    },
    "not_applicable_cases": [
      "active__autolink",
      "active__data-uri",
      "active__markdown-image",
      "active__markdown-link",
      "active__raw-html",
      "active__reference-link"
    ]
  }
}