First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
93 lines
3.8 KiB
JSON
93 lines
3.8 KiB
JSON
{
|
|
"schema": "okf-wiki/wholedoc-manifest/1",
|
|
"thresholds": {
|
|
"span_chars": [
|
|
20,
|
|
400
|
|
],
|
|
"key_chars": [
|
|
3,
|
|
60
|
|
],
|
|
"span_occurrences": "exactly 1 in its file (so its position is defined)",
|
|
"whole_min_file_lines": 150,
|
|
"whole_min_keys": 3,
|
|
"whole_min_parts": 3,
|
|
"whole_part_gap": 0.15,
|
|
"whole_part_rule": "spans sorted by start offset; a new part begins where the gap between two starts is >= whole_part_gap x file length (characters)",
|
|
"whole_coverage": "first span starts before 1/3 of the file, last span starts at or after 2/3",
|
|
"whole_one_span": "no single span holds every answer key (case-folded)",
|
|
"count_min": 5,
|
|
"count_rule": "count == number of evidence spans == matches of item_pattern in the file; every match starts inside exactly one span",
|
|
"count_not_verbatim": "the count stands nowhere in the file as a number (not part of a longer number, version or identifier) nor as an English number word",
|
|
"cut_max_files": 20
|
|
},
|
|
"cut": {
|
|
"rule": "E2 dev questions sorted by id; repeatedly the question needing the fewest NEW source files (ties: lowest id), while the union stays <= max_files",
|
|
"e2_set_sha256": "bc98c9a16d18f7a851c2ce319a32325244076f8be30ea3d8266bdea7a21aad68",
|
|
"max_files": 20,
|
|
"questions": 7,
|
|
"files": 19,
|
|
"names_sha256": "a1beefea5e90048ff14b21c466cb35e5b94f2b0d894e1e154ab03b9ccd133af6",
|
|
"names": [
|
|
"docs-en-microsoft-foundry.md",
|
|
"docs-en-permission-modes.md",
|
|
"docs-en-setup.md",
|
|
"release-v0.2.34.md",
|
|
"release-v0.2.37.md",
|
|
"release-v2.1.111.md",
|
|
"release-v2.1.118.md",
|
|
"release-v2.1.136.md",
|
|
"release-v2.1.152.md",
|
|
"release-v2.1.163.md",
|
|
"release-v2.1.2.md",
|
|
"release-v2.1.208.md",
|
|
"release-v2.1.267.md",
|
|
"release-v2.1.274.md",
|
|
"release-v2.1.76.md",
|
|
"release-v2.1.83.md",
|
|
"release-v2.1.92.md",
|
|
"release-v2.1.94.md",
|
|
"release-v2.1.98.md"
|
|
]
|
|
},
|
|
"sources": {
|
|
"files": 599,
|
|
"tree_sha256": "6daf3ed73335f3e6f332c7b188adb5aebbc659b6fdd1d7591280e129648a06d5"
|
|
},
|
|
"seed": 20260927,
|
|
"dev": {
|
|
"file": "gates-local/wholedoc/set-12.json",
|
|
"sha256": "62e44978485da0907467cd187b21a0495d4d47ce31a5555bbb383ba9995bd4da",
|
|
"questions": 12,
|
|
"whole": 6,
|
|
"count": 6
|
|
},
|
|
"held_out": {
|
|
"sha256": "9d468c215950d8a700919d3ee1bdab66a5e8447089d4b21e6e1d0df7d355afae",
|
|
"questions": 4,
|
|
"whole": 2,
|
|
"count": 2
|
|
},
|
|
"draw": "rng = random.Random(seed); for type in ('count', 'whole'): rng.sample(sorted(ids of type), held_out_per_type); both parts renumbered d../h.. in sorted original-id order",
|
|
"keys_rewritten": {
|
|
"order": "E2c",
|
|
"rule": "answer keys rewritten as facts (gates/keyform.py): every key of two or more words with a function word outside backticks became the runs of its own words without the function words, each still inside its span; questions, files, spans, counts and patterns unchanged",
|
|
"before": {
|
|
"dev_sha256": "77338b7c0aa09a0b5ab33cc6e050f33be06182d061c2011f177d6f4a596ab3b2",
|
|
"held_out_sha256": "2fe50335e2decccfb821707c50aa6a0e611ee1f88fd536348468b2532072ace0",
|
|
"cut_e2_set_sha256": "342686e100f2016e376bc4ecb28c14657646d1c35ba8e3bcad3593f4204a001a"
|
|
},
|
|
"cut": "recomputed from the rewritten E2 dev set: same names (names_sha256 unchanged), so only e2_set_sha256 moved"
|
|
},
|
|
"negated_marked": {
|
|
"order": "E2d",
|
|
"rule": "\"negated\": true added to the polarity keys of E2c's list that stand negated in their own spans (gates/multidoc.py negated_in): 17 of 22; nothing else changed",
|
|
"before": {
|
|
"dev_sha256": "6ecb9d624e0387fcc6ac466ac724fdb5f7196ff162405859c20665fbe87b9f9c",
|
|
"held_out_sha256": "e73e3963d2dd3e0947ad798e4aaa1320b4fe60025a6064a653463c31c7161a03",
|
|
"cut_e2_set_sha256": "b4832f93d10090d439554d1a159a5140b03eecf6eb453faafa6638aec50f23b0"
|
|
},
|
|
"cut": "recomputed from the marked E2 dev set: same names (names_sha256 unchanged), so only e2_set_sha256 moved"
|
|
}
|
|
}
|