Kjell Tore Guttormsen
49fd18c6d2
compute-base-rate.mjs (tynn CLI over testet lib/base-rate.mjs) -> base-rate-report.{json,md}.
373 paastander: verifiserbar feilrate 13,4 % (40/299), Wilson [10,0 %, 17,7 %]; unsourced 19,8 %.
staleness-recall 0/40 = 0 % -> korrekthets-judge er eneste mekanisme som fanger feilene.
Feil konsentrert i fetchbare typer (sku 36 %, version 25 %, tpm 20 %); pris 74 % unsourced -> operatoer-gated.
GATE besluttet i plan-doc: BYGG Fase 3 scoped til fetchbare claim_types (taxonomy/status/version/tpm/sku/region); pris ikke maskinverifiserbar.
333 lines
7.7 KiB
JSON
333 lines
7.7 KiB
JSON
{
|
|
"_meta": {
|
|
"source": "gold-correctness-set.json",
|
|
"claim_count": 373
|
|
},
|
|
"overall": {
|
|
"total": 373,
|
|
"byVerdict": {
|
|
"correct": 259,
|
|
"outdated": 29,
|
|
"wrong": 11,
|
|
"unsourced": 74
|
|
},
|
|
"errors": 40,
|
|
"verifiable": 299,
|
|
"unsourced": 74,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 40,
|
|
"errorRate": 0.13377926421404682,
|
|
"errorRateWilson": {
|
|
"p": 0.13377926421404682,
|
|
"low": 0.0998039899711332,
|
|
"high": 0.17704568986149216
|
|
},
|
|
"unsourcedRate": 0.19839142091152814
|
|
},
|
|
"bySkill": {
|
|
"ms-ai-advisor": {
|
|
"total": 79,
|
|
"byVerdict": {
|
|
"correct": 59,
|
|
"outdated": 7,
|
|
"wrong": 2,
|
|
"unsourced": 11
|
|
},
|
|
"errors": 9,
|
|
"verifiable": 68,
|
|
"unsourced": 11,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 9,
|
|
"errorRate": 0.1323529411764706,
|
|
"errorRateWilson": {
|
|
"p": 0.1323529411764706,
|
|
"low": 0.07122163958543326,
|
|
"high": 0.2328027696704853
|
|
},
|
|
"unsourcedRate": 0.13924050632911392
|
|
},
|
|
"ms-ai-engineering": {
|
|
"total": 86,
|
|
"byVerdict": {
|
|
"correct": 54,
|
|
"outdated": 9,
|
|
"wrong": 4,
|
|
"unsourced": 19
|
|
},
|
|
"errors": 13,
|
|
"verifiable": 67,
|
|
"unsourced": 19,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 13,
|
|
"errorRate": 0.19402985074626866,
|
|
"errorRateWilson": {
|
|
"p": 0.19402985074626866,
|
|
"low": 0.11705063857457099,
|
|
"high": 0.3041933762415822
|
|
},
|
|
"unsourcedRate": 0.22093023255813954
|
|
},
|
|
"ms-ai-governance": {
|
|
"total": 76,
|
|
"byVerdict": {
|
|
"correct": 52,
|
|
"outdated": 5,
|
|
"wrong": 2,
|
|
"unsourced": 17
|
|
},
|
|
"errors": 7,
|
|
"verifiable": 59,
|
|
"unsourced": 17,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 7,
|
|
"errorRate": 0.11864406779661017,
|
|
"errorRateWilson": {
|
|
"p": 0.11864406779661017,
|
|
"low": 0.058675082779140006,
|
|
"high": 0.22523875773415053
|
|
},
|
|
"unsourcedRate": 0.2236842105263158
|
|
},
|
|
"ms-ai-infrastructure": {
|
|
"total": 36,
|
|
"byVerdict": {
|
|
"correct": 29,
|
|
"outdated": 1,
|
|
"wrong": 2,
|
|
"unsourced": 4
|
|
},
|
|
"errors": 3,
|
|
"verifiable": 32,
|
|
"unsourced": 4,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 3,
|
|
"errorRate": 0.09375,
|
|
"errorRateWilson": {
|
|
"p": 0.09375,
|
|
"low": 0.032400962626319516,
|
|
"high": 0.24218499335778831
|
|
},
|
|
"unsourcedRate": 0.1111111111111111
|
|
},
|
|
"ms-ai-security": {
|
|
"total": 96,
|
|
"byVerdict": {
|
|
"correct": 65,
|
|
"outdated": 7,
|
|
"wrong": 1,
|
|
"unsourced": 23
|
|
},
|
|
"errors": 8,
|
|
"verifiable": 73,
|
|
"unsourced": 23,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 8,
|
|
"errorRate": 0.1095890410958904,
|
|
"errorRateWilson": {
|
|
"p": 0.1095890410958904,
|
|
"low": 0.0565860416101191,
|
|
"high": 0.20162825897706282
|
|
},
|
|
"unsourcedRate": 0.23958333333333334
|
|
}
|
|
},
|
|
"byStratum": {
|
|
"volatile": {
|
|
"total": 331,
|
|
"byVerdict": {
|
|
"correct": 222,
|
|
"outdated": 28,
|
|
"wrong": 10,
|
|
"unsourced": 71
|
|
},
|
|
"errors": 38,
|
|
"verifiable": 260,
|
|
"unsourced": 71,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 38,
|
|
"errorRate": 0.14615384615384616,
|
|
"errorRateWilson": {
|
|
"p": 0.14615384615384616,
|
|
"low": 0.1083692375274728,
|
|
"high": 0.1942426326249217
|
|
},
|
|
"unsourcedRate": 0.21450151057401812
|
|
},
|
|
"control": {
|
|
"total": 42,
|
|
"byVerdict": {
|
|
"correct": 37,
|
|
"outdated": 1,
|
|
"wrong": 1,
|
|
"unsourced": 3
|
|
},
|
|
"errors": 2,
|
|
"verifiable": 39,
|
|
"unsourced": 3,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 2,
|
|
"errorRate": 0.05128205128205128,
|
|
"errorRateWilson": {
|
|
"p": 0.05128205128205128,
|
|
"low": 0.014177657646399527,
|
|
"high": 0.1688593904563791
|
|
},
|
|
"unsourcedRate": 0.07142857142857142
|
|
}
|
|
},
|
|
"byClaimType": {
|
|
"version": {
|
|
"total": 33,
|
|
"byVerdict": {
|
|
"correct": 24,
|
|
"outdated": 6,
|
|
"wrong": 2,
|
|
"unsourced": 1
|
|
},
|
|
"errors": 8,
|
|
"verifiable": 32,
|
|
"unsourced": 1,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 8,
|
|
"errorRate": 0.25,
|
|
"errorRateWilson": {
|
|
"p": 0.25,
|
|
"low": 0.13252243982621553,
|
|
"high": 0.4210689177024662
|
|
},
|
|
"unsourcedRate": 0.030303030303030304
|
|
},
|
|
"tpm": {
|
|
"total": 30,
|
|
"byVerdict": {
|
|
"correct": 20,
|
|
"outdated": 4,
|
|
"wrong": 1,
|
|
"unsourced": 5
|
|
},
|
|
"errors": 5,
|
|
"verifiable": 25,
|
|
"unsourced": 5,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 5,
|
|
"errorRate": 0.2,
|
|
"errorRateWilson": {
|
|
"p": 0.2,
|
|
"low": 0.08860454100652485,
|
|
"high": 0.3913133553653825
|
|
},
|
|
"unsourcedRate": 0.16666666666666666
|
|
},
|
|
"region": {
|
|
"total": 20,
|
|
"byVerdict": {
|
|
"correct": 15,
|
|
"outdated": 2,
|
|
"wrong": 0,
|
|
"unsourced": 3
|
|
},
|
|
"errors": 2,
|
|
"verifiable": 17,
|
|
"unsourced": 3,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 2,
|
|
"errorRate": 0.11764705882352941,
|
|
"errorRateWilson": {
|
|
"p": 0.11764705882352941,
|
|
"low": 0.03287908001292092,
|
|
"high": 0.3433684249770991
|
|
},
|
|
"unsourcedRate": 0.15
|
|
},
|
|
"status": {
|
|
"total": 70,
|
|
"byVerdict": {
|
|
"correct": 58,
|
|
"outdated": 4,
|
|
"wrong": 3,
|
|
"unsourced": 5
|
|
},
|
|
"errors": 7,
|
|
"verifiable": 65,
|
|
"unsourced": 5,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 7,
|
|
"errorRate": 0.1076923076923077,
|
|
"errorRateWilson": {
|
|
"p": 0.1076923076923077,
|
|
"low": 0.05315354431925606,
|
|
"high": 0.20601533031468616
|
|
},
|
|
"unsourcedRate": 0.07142857142857142
|
|
},
|
|
"price": {
|
|
"total": 76,
|
|
"byVerdict": {
|
|
"correct": 20,
|
|
"outdated": 0,
|
|
"wrong": 0,
|
|
"unsourced": 56
|
|
},
|
|
"errors": 0,
|
|
"verifiable": 20,
|
|
"unsourced": 56,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 0,
|
|
"errorRate": 0,
|
|
"errorRateWilson": {
|
|
"p": 0,
|
|
"low": 0,
|
|
"high": 0.16113012549493322
|
|
},
|
|
"unsourcedRate": 0.7368421052631579
|
|
},
|
|
"taxonomy": {
|
|
"total": 122,
|
|
"byVerdict": {
|
|
"correct": 108,
|
|
"outdated": 7,
|
|
"wrong": 3,
|
|
"unsourced": 4
|
|
},
|
|
"errors": 10,
|
|
"verifiable": 118,
|
|
"unsourced": 4,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 10,
|
|
"errorRate": 0.0847457627118644,
|
|
"errorRateWilson": {
|
|
"p": 0.0847457627118644,
|
|
"low": 0.046682191106169606,
|
|
"high": 0.14899481904471476
|
|
},
|
|
"unsourcedRate": 0.03278688524590164
|
|
},
|
|
"sku": {
|
|
"total": 22,
|
|
"byVerdict": {
|
|
"correct": 14,
|
|
"outdated": 6,
|
|
"wrong": 2,
|
|
"unsourced": 0
|
|
},
|
|
"errors": 8,
|
|
"verifiable": 22,
|
|
"unsourced": 0,
|
|
"errorsStalenessCatchable": 0,
|
|
"errorsJudgeUnique": 8,
|
|
"errorRate": 0.36363636363636365,
|
|
"errorRateWilson": {
|
|
"p": 0.36363636363636365,
|
|
"low": 0.19732972772607899,
|
|
"high": 0.5704865065628195
|
|
},
|
|
"unsourcedRate": 0
|
|
}
|
|
},
|
|
"_verdicts": [
|
|
"correct",
|
|
"outdated",
|
|
"wrong",
|
|
"unsourced"
|
|
]
|
|
}
|