open/ms-ai-architect
1
0
Fork 0
Code Issues Releases 1 Activity

feat(ms-ai-architect): Spor 2b gull-rekonsiliering — 12 uenigheter løst, fasit herdet (P 84→92%, R 84→88%) [skip-docs]

Browse source 
4 gull-feil rettet (FP1/FP2/FP6/FN1) + 1 note-fiks (FP4); 8 judge-feil dokumentert som kalibreringsmål for Spor 2a/judge-prompt-v3. Hver endring live-belagt mot MS Learn. Nedre-grense-policy vedtatt. 0 uløste uenigheter. Suite 637/637. Logg: docs/ref-kb-gold-reconciliation-2026-06.md; herdet rapport: judge-bakeoff-report-v2-reconciled (frozen v2 beholdt).
This commit is contained in:
Kjell Tore Guttormsen 2026-06-30 07:20:21 +02:00
commit 4ab1138760
5 changed files with 404 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

234
scripts/kb-eval/data/judge-bakeoff-report-v2-reconciled.json Normal file
View file

@ -0,0 +1,234 @@
{
"_meta": {
"source": "gold-correctness-set.json + judge-bakeoff-results.json",
"thresholds": {
"minRecall": 0.8,
"minPrecision": 0.7
},
"judged": 255
},
"population": {
"total": 255,
"verifiable": 240,
"positives": 40,
"negatives": 200,
"unsourcedInP": 15
},
"arms": {
"staleness": {
"tp": 0,
"fp": 0,
"fn": 40,
"tn": 200,
"positives": 40,
"negatives": 200,
"flagged": 0,
"precision": null,
"recall": 0,
"f1": null,
"recallWilson": {
"p": 0,
"low": 0,
"high": 0.08762453925039232
},
"precisionWilson": null
},
"judge": {
"tp": 35,
"fp": 3,
"fn": 5,
"tn": 197,
"positives": 40,
"negatives": 200,
"flagged": 38,
"precision": 0.9210526315789473,
"recall": 0.875,
"f1": 0.8974358974358975,
"recallWilson": {
"p": 0.875,
"low": 0.7388757932976187,
"high": 0.9454058022645873
},
"precisionWilson": {
"p": 0.9210526315789473,
"low": 0.792003210797347,
"high": 0.972785898605735
}
},
"hybrid": {
"tp": 35,
"fp": 3,
"fn": 5,
"tn": 197,
"positives": 40,
"negatives": 200,
"flagged": 38,
"precision": 0.9210526315789473,
"recall": 0.875,
"f1": 0.8974358974358975,
"recallWilson": {
"p": 0.875,
"low": 0.7388757932976187,
"high": 0.9454058022645873
},
"precisionWilson": {
"p": 0.9210526315789473,
"low": 0.792003210797347,
"high": 0.972785898605735
}
}
},
"sourceSilent": {
"onVerifiableNegative": 3,
"onVerifiableError": 2,
"agreesWithUnsourced": 5,
"disagreesWithUnsourced": 10
},
"byClaimType": {
"version": {
"tp": 6,
"fp": 0,
"fn": 1,
"tn": 21,
"positives": 7,
"negatives": 21,
"flagged": 6,
"precision": 1,
"recall": 0.8571428571428571,
"f1": 0.923076923076923,
"recallWilson": {
"p": 0.8571428571428571,
"low": 0.4868654966809701,
"high": 0.9743210440510252
},
"precisionWilson": {
"p": 1,
"low": 0.6096569663469354,
"high": 0.9999999999999999
}
},
"tpm": {
"tp": 4,
"fp": 0,
"fn": 1,
"tn": 20,
"positives": 5,
"negatives": 20,
"flagged": 4,
"precision": 1,
"recall": 0.8,
"f1": 0.888888888888889,
"recallWilson": {
"p": 0.8,
"low": 0.3755282641185388,
"high": 0.9637768390302125
},
"precisionWilson": {
"p": 1,
"low": 0.5100999795960008,
"high": 1
}
},
"region": {
"tp": 1,
"fp": 0,
"fn": 0,
"tn": 14,
"positives": 1,
"negatives": 14,
"flagged": 1,
"precision": 1,
"recall": 1,
"f1": 1,
"recallWilson": {
"p": 1,
"low": 0.2065432914738929,
"high": 1
},
"precisionWilson": {
"p": 1,
"low": 0.2065432914738929,
"high": 1
}
},
"status": {
"tp": 7,
"fp": 0,
"fn": 1,
"tn": 45,
"positives": 8,
"negatives": 45,
"flagged": 7,
"precision": 1,
"recall": 0.875,
"f1": 0.9333333333333333,
"recallWilson": {
"p": 0.875,
"low": 0.5291051942301386,
"high": 0.9775830911367038
},
"precisionWilson": {
"p": 1,
"low": 0.6456611570247934,
"high": 1
}
},
"taxonomy": {
"tp": 11,
"fp": 3,
"fn": 0,
"tn": 83,
"positives": 11,
"negatives": 86,
"flagged": 14,
"precision": 0.7857142857142857,
"recall": 1,
"f1": 0.88,
"recallWilson": {
"p": 1,
"low": 0.7411599827511859,
"high": 1
},
"precisionWilson": {
"p": 0.7857142857142857,
"low": 0.5241027622679172,
"high": 0.9242875166308363
}
},
"sku": {
"tp": 6,
"fp": 0,
"fn": 2,
"tn": 14,
"positives": 8,
"negatives": 14,
"flagged": 6,
"precision": 1,
"recall": 0.75,
"f1": 0.8571428571428571,
"recallWilson": {
"p": 0.75,
"low": 0.40926987910258916,
"high": 0.9285223111419724
},
"precisionWilson": {
"p": 1,
"low": 0.6096569663469354,
"high": 0.9999999999999999
}
}
},
"gate": {
"pass": true,
"recallOk": true,
"precisionOk": true,
"beatsStaleness": true,
"thresholds": {
"minRecall": 0.8,
"minPrecision": 0.7
},
"reasons": [
"all criteria met"
]
}
}