First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
145 lines
3.5 KiB
JSON
145 lines
3.5 KiB
JSON
{
|
|
"schema": "okf-wiki/calibration-manifest/2",
|
|
"seed": 20261008,
|
|
"shuffle": "random.Random(seed): each label's pairs sorted by draft id and shuffled; the first `lone` unsupported pairs alone in a call at positions spread evenly over 1-10, the rest `dense` to a call; counted supported pairs one per call in call order; filler pairs fill every call to ten in a cycle, never twice in a call and never beside the unsupported pair they are the twin of; the other positions shuffled; ids renumbered c001.. in call order, calls k001..",
|
|
"set": {
|
|
"file": "gates-local/calibration/pairs-300.json",
|
|
"sha256": "fd0c9c975bafd8770bab749e1faf9b38b0711491a4f9f433b40ef2004b4c27f7",
|
|
"pairs": 1400
|
|
},
|
|
"key": {
|
|
"file": "gates-local/calibration/key-300.json",
|
|
"sha256": "97c6fada731d5718b94a543867d1adac81547e96072e27fc3c0c7fae324fb4d2"
|
|
},
|
|
"calls": {
|
|
"file": "gates-local/calibration/calls-300.json",
|
|
"sha256": "a356abdab4e0eb868fbcdb77e0a634c1bfa28648c5a5d6af1ac5a09748e52709",
|
|
"calls": 140,
|
|
"call_size_max": 10,
|
|
"unsupported_alone": 100,
|
|
"unsupported_per_call": {
|
|
"1": 100,
|
|
"5": 40
|
|
},
|
|
"built_with": {
|
|
"lone": 100,
|
|
"dense": 5
|
|
}
|
|
},
|
|
"counts": {
|
|
"unsupported": 300,
|
|
"supported": 50,
|
|
"filler-supported": 1050,
|
|
"filler_distinct": 250,
|
|
"by_kind": {
|
|
"overgeneralisation": 50,
|
|
"wrong-number": 50,
|
|
"swapped-cause": 50,
|
|
"not-mentioned": 50,
|
|
"reversed-polarity": 50,
|
|
"conflated-sources": 50,
|
|
"paraphrase": 15,
|
|
"inference": 20,
|
|
"synthesis": 15
|
|
},
|
|
"by_position": {
|
|
"1": {
|
|
"unsupported": 33,
|
|
"supported": 8,
|
|
"filler-supported": 99
|
|
},
|
|
"2": {
|
|
"unsupported": 30,
|
|
"supported": 5,
|
|
"filler-supported": 105
|
|
},
|
|
"3": {
|
|
"unsupported": 28,
|
|
"supported": 6,
|
|
"filler-supported": 106
|
|
},
|
|
"4": {
|
|
"unsupported": 23,
|
|
"supported": 5,
|
|
"filler-supported": 112
|
|
},
|
|
"5": {
|
|
"unsupported": 32,
|
|
"supported": 6,
|
|
"filler-supported": 102
|
|
},
|
|
"6": {
|
|
"unsupported": 23,
|
|
"supported": 4,
|
|
"filler-supported": 113
|
|
},
|
|
"7": {
|
|
"unsupported": 30,
|
|
"supported": 6,
|
|
"filler-supported": 104
|
|
},
|
|
"8": {
|
|
"unsupported": 24,
|
|
"supported": 3,
|
|
"filler-supported": 113
|
|
},
|
|
"9": {
|
|
"unsupported": 39,
|
|
"supported": 1,
|
|
"filler-supported": 100
|
|
},
|
|
"10": {
|
|
"unsupported": 38,
|
|
"supported": 6,
|
|
"filler-supported": 96
|
|
}
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"caught_min": 300,
|
|
"accepted_min": 45
|
|
},
|
|
"sources": {
|
|
"files": 599,
|
|
"tree_sha256": "6daf3ed73335f3e6f332c7b188adb5aebbc659b6fdd1d7591280e129648a06d5"
|
|
},
|
|
"history": [
|
|
{
|
|
"schema": "okf-wiki/calibration-manifest/1",
|
|
"seed": 20260927,
|
|
"shuffle": "random.Random(seed).shuffle(pairs sorted by draft id); renumbered c001..",
|
|
"set": {
|
|
"file": "gates-local/calibration/pairs-100.json",
|
|
"sha256": "bb9d303840dd43eff8c55304c9ce9d935922678a84fe97022c6c721e9fa61d0d",
|
|
"pairs": 100
|
|
},
|
|
"key": {
|
|
"file": "gates-local/calibration/key-100.json",
|
|
"sha256": "b30a37019dfe224474a0cf1b1aa4a82ccda2c332a035ecd4fb1b4500f78343ca"
|
|
},
|
|
"counts": {
|
|
"unsupported": 50,
|
|
"supported": 50,
|
|
"by_kind": {
|
|
"overgeneralisation": 9,
|
|
"wrong-number": 9,
|
|
"swapped-cause": 8,
|
|
"not-mentioned": 8,
|
|
"reversed-polarity": 8,
|
|
"conflated-sources": 8,
|
|
"paraphrase": 15,
|
|
"inference": 20,
|
|
"synthesis": 15
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"caught_min": 50,
|
|
"accepted_min": 45
|
|
},
|
|
"sources": {
|
|
"files": 599,
|
|
"tree_sha256": "6daf3ed73335f3e6f332c7b188adb5aebbc659b6fdd1d7591280e129648a06d5"
|
|
}
|
|
}
|
|
]
|
|
}
|