Kjell Tore Guttormsen
79e34312ea
Blind fan-out: 15 batcher Opus 4.8 xhigh, 255 P-påstander dekket (0 missing/dupes/extra), hver judget mot live MS Learn via microsoft_docs_fetch (judgen så aldri gull-verdict). Resultat mot forhåndsregistrert gate (recall >=0.80, presisjon >=0.70): - judge: presisjon 81.3% (PASS), recall 68.4% (FAIL, Wilson [52.5-80.9%]), F1 0.743, 26/38 fanget - staleness baseline: recall 0/38 (judge slår den desisivt) - recall-drag konsentrert: sku 37.5%, taxonomy 66.7%; version/status/tpm allerede 80-86% - 10/12 bommer var grounded-men-feil (brittle dekomponering, isolert til sku/taxonomy) Per pre-registrert gate: STOPP, bygg IKKE S3. Fork (operatør) registrert i plan-doc: (a) honorer FAIL, (b) scoped judge, (c) målrettet iterasjon + re-kjør.
234 lines
4.9 KiB
JSON
234 lines
4.9 KiB
JSON
{
|
|
"_meta": {
|
|
"source": "gold-correctness-set.json + judge-bakeoff-results.json",
|
|
"thresholds": {
|
|
"minRecall": 0.8,
|
|
"minPrecision": 0.7
|
|
},
|
|
"judged": 255
|
|
},
|
|
"population": {
|
|
"total": 255,
|
|
"verifiable": 240,
|
|
"positives": 38,
|
|
"negatives": 202,
|
|
"unsourcedInP": 15
|
|
},
|
|
"arms": {
|
|
"staleness": {
|
|
"tp": 0,
|
|
"fp": 0,
|
|
"fn": 38,
|
|
"tn": 202,
|
|
"positives": 38,
|
|
"negatives": 202,
|
|
"flagged": 0,
|
|
"precision": null,
|
|
"recall": 0,
|
|
"f1": null,
|
|
"recallWilson": {
|
|
"p": 0,
|
|
"low": 0,
|
|
"high": 0.09181293258383999
|
|
},
|
|
"precisionWilson": null
|
|
},
|
|
"judge": {
|
|
"tp": 26,
|
|
"fp": 6,
|
|
"fn": 12,
|
|
"tn": 196,
|
|
"positives": 38,
|
|
"negatives": 202,
|
|
"flagged": 32,
|
|
"precision": 0.8125,
|
|
"recall": 0.6842105263157895,
|
|
"f1": 0.742857142857143,
|
|
"recallWilson": {
|
|
"p": 0.6842105263157895,
|
|
"low": 0.5254394356005149,
|
|
"high": 0.8091557997633335
|
|
},
|
|
"precisionWilson": {
|
|
"p": 0.8125,
|
|
"low": 0.6469050424507756,
|
|
"high": 0.9111057606383721
|
|
}
|
|
},
|
|
"hybrid": {
|
|
"tp": 26,
|
|
"fp": 6,
|
|
"fn": 12,
|
|
"tn": 196,
|
|
"positives": 38,
|
|
"negatives": 202,
|
|
"flagged": 32,
|
|
"precision": 0.8125,
|
|
"recall": 0.6842105263157895,
|
|
"f1": 0.742857142857143,
|
|
"recallWilson": {
|
|
"p": 0.6842105263157895,
|
|
"low": 0.5254394356005149,
|
|
"high": 0.8091557997633335
|
|
},
|
|
"precisionWilson": {
|
|
"p": 0.8125,
|
|
"low": 0.6469050424507756,
|
|
"high": 0.9111057606383721
|
|
}
|
|
}
|
|
},
|
|
"sourceSilent": {
|
|
"onVerifiableNegative": 0,
|
|
"onVerifiableError": 2,
|
|
"agreesWithUnsourced": 5,
|
|
"disagreesWithUnsourced": 10
|
|
},
|
|
"byClaimType": {
|
|
"version": {
|
|
"tp": 6,
|
|
"fp": 0,
|
|
"fn": 1,
|
|
"tn": 21,
|
|
"positives": 7,
|
|
"negatives": 21,
|
|
"flagged": 6,
|
|
"precision": 1,
|
|
"recall": 0.8571428571428571,
|
|
"f1": 0.923076923076923,
|
|
"recallWilson": {
|
|
"p": 0.8571428571428571,
|
|
"low": 0.4868654966809701,
|
|
"high": 0.9743210440510252
|
|
},
|
|
"precisionWilson": {
|
|
"p": 1,
|
|
"low": 0.6096569663469354,
|
|
"high": 0.9999999999999999
|
|
}
|
|
},
|
|
"tpm": {
|
|
"tp": 4,
|
|
"fp": 0,
|
|
"fn": 1,
|
|
"tn": 20,
|
|
"positives": 5,
|
|
"negatives": 20,
|
|
"flagged": 4,
|
|
"precision": 1,
|
|
"recall": 0.8,
|
|
"f1": 0.888888888888889,
|
|
"recallWilson": {
|
|
"p": 0.8,
|
|
"low": 0.3755282641185388,
|
|
"high": 0.9637768390302125
|
|
},
|
|
"precisionWilson": {
|
|
"p": 1,
|
|
"low": 0.5100999795960008,
|
|
"high": 1
|
|
}
|
|
},
|
|
"region": {
|
|
"tp": 1,
|
|
"fp": 0,
|
|
"fn": 1,
|
|
"tn": 13,
|
|
"positives": 2,
|
|
"negatives": 13,
|
|
"flagged": 1,
|
|
"precision": 1,
|
|
"recall": 0.5,
|
|
"f1": 0.6666666666666666,
|
|
"recallWilson": {
|
|
"p": 0.5,
|
|
"low": 0.09452865480086614,
|
|
"high": 0.9054713451991339
|
|
},
|
|
"precisionWilson": {
|
|
"p": 1,
|
|
"low": 0.2065432914738929,
|
|
"high": 1
|
|
}
|
|
},
|
|
"status": {
|
|
"tp": 6,
|
|
"fp": 2,
|
|
"fn": 1,
|
|
"tn": 44,
|
|
"positives": 7,
|
|
"negatives": 46,
|
|
"flagged": 8,
|
|
"precision": 0.75,
|
|
"recall": 0.8571428571428571,
|
|
"f1": 0.7999999999999999,
|
|
"recallWilson": {
|
|
"p": 0.8571428571428571,
|
|
"low": 0.4868654966809701,
|
|
"high": 0.9743210440510252
|
|
},
|
|
"precisionWilson": {
|
|
"p": 0.75,
|
|
"low": 0.40926987910258916,
|
|
"high": 0.9285223111419724
|
|
}
|
|
},
|
|
"taxonomy": {
|
|
"tp": 6,
|
|
"fp": 4,
|
|
"fn": 3,
|
|
"tn": 84,
|
|
"positives": 9,
|
|
"negatives": 88,
|
|
"flagged": 10,
|
|
"precision": 0.6,
|
|
"recall": 0.6666666666666666,
|
|
"f1": 0.631578947368421,
|
|
"recallWilson": {
|
|
"p": 0.6666666666666666,
|
|
"low": 0.3541973474990897,
|
|
"high": 0.8794184013172571
|
|
},
|
|
"precisionWilson": {
|
|
"p": 0.6,
|
|
"low": 0.3126695474501863,
|
|
"high": 0.8318224187964902
|
|
}
|
|
},
|
|
"sku": {
|
|
"tp": 3,
|
|
"fp": 0,
|
|
"fn": 5,
|
|
"tn": 14,
|
|
"positives": 8,
|
|
"negatives": 14,
|
|
"flagged": 3,
|
|
"precision": 1,
|
|
"recall": 0.375,
|
|
"f1": 0.5454545454545454,
|
|
"recallWilson": {
|
|
"p": 0.375,
|
|
"low": 0.13684175946152105,
|
|
"high": 0.6942621454161981
|
|
},
|
|
"precisionWilson": {
|
|
"p": 1,
|
|
"low": 0.43849391955098227,
|
|
"high": 1
|
|
}
|
|
}
|
|
},
|
|
"gate": {
|
|
"pass": false,
|
|
"recallOk": false,
|
|
"precisionOk": true,
|
|
"beatsStaleness": true,
|
|
"thresholds": {
|
|
"minRecall": 0.8,
|
|
"minPrecision": 0.7
|
|
},
|
|
"reasons": [
|
|
"recall 0.684 < minRecall 0.8"
|
|
]
|
|
}
|
|
}
|