open/ms-ai-architect
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity Actions

feat(ms-ai-architect): Fase 0 steg 3 — base-rate-rapport + GATE: BYGG Fase 3 (scoped/hybrid) [skip-docs]

Browse source 
compute-base-rate.mjs (tynn CLI over testet lib/base-rate.mjs) -> base-rate-report.{json,md}.
373 paastander: verifiserbar feilrate 13,4 % (40/299), Wilson [10,0 %, 17,7 %]; unsourced 19,8 %.
staleness-recall 0/40 = 0 % -> korrekthets-judge er eneste mekanisme som fanger feilene.
Feil konsentrert i fetchbare typer (sku 36 %, version 25 %, tpm 20 %); pris 74 % unsourced -> operatoer-gated.
GATE besluttet i plan-doc: BYGG Fase 3 scoped til fetchbare claim_types (taxonomy/status/version/tpm/sku/region); pris ikke maskinverifiserbar.
This commit is contained in:
Kjell Tore Guttormsen 2026-06-26 17:15:54 +02:00
commit 49fd18c6d2
4 changed files with 512 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

333
scripts/kb-eval/data/base-rate-report.json Normal file
View file

@ -0,0 +1,333 @@
{
"_meta": {
"source": "gold-correctness-set.json",
"claim_count": 373
},
"overall": {
"total": 373,
"byVerdict": {
"correct": 259,
"outdated": 29,
"wrong": 11,
"unsourced": 74
},
"errors": 40,
"verifiable": 299,
"unsourced": 74,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 40,
"errorRate": 0.13377926421404682,
"errorRateWilson": {
"p": 0.13377926421404682,
"low": 0.0998039899711332,
"high": 0.17704568986149216
},
"unsourcedRate": 0.19839142091152814
},
"bySkill": {
"ms-ai-advisor": {
"total": 79,
"byVerdict": {
"correct": 59,
"outdated": 7,
"wrong": 2,
"unsourced": 11
},
"errors": 9,
"verifiable": 68,
"unsourced": 11,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 9,
"errorRate": 0.1323529411764706,
"errorRateWilson": {
"p": 0.1323529411764706,
"low": 0.07122163958543326,
"high": 0.2328027696704853
},
"unsourcedRate": 0.13924050632911392
},
"ms-ai-engineering": {
"total": 86,
"byVerdict": {
"correct": 54,
"outdated": 9,
"wrong": 4,
"unsourced": 19
},
"errors": 13,
"verifiable": 67,
"unsourced": 19,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 13,
"errorRate": 0.19402985074626866,
"errorRateWilson": {
"p": 0.19402985074626866,
"low": 0.11705063857457099,
"high": 0.3041933762415822
},
"unsourcedRate": 0.22093023255813954
},
"ms-ai-governance": {
"total": 76,
"byVerdict": {
"correct": 52,
"outdated": 5,
"wrong": 2,
"unsourced": 17
},
"errors": 7,
"verifiable": 59,
"unsourced": 17,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 7,
"errorRate": 0.11864406779661017,
"errorRateWilson": {
"p": 0.11864406779661017,
"low": 0.058675082779140006,
"high": 0.22523875773415053
},
"unsourcedRate": 0.2236842105263158
},
"ms-ai-infrastructure": {
"total": 36,
"byVerdict": {
"correct": 29,
"outdated": 1,
"wrong": 2,
"unsourced": 4
},
"errors": 3,
"verifiable": 32,
"unsourced": 4,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 3,
"errorRate": 0.09375,
"errorRateWilson": {
"p": 0.09375,
"low": 0.032400962626319516,
"high": 0.24218499335778831
},
"unsourcedRate": 0.1111111111111111
},
"ms-ai-security": {
"total": 96,
"byVerdict": {
"correct": 65,
"outdated": 7,
"wrong": 1,
"unsourced": 23
},
"errors": 8,
"verifiable": 73,
"unsourced": 23,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 8,
"errorRate": 0.1095890410958904,
"errorRateWilson": {
"p": 0.1095890410958904,
"low": 0.0565860416101191,
"high": 0.20162825897706282
},
"unsourcedRate": 0.23958333333333334
}
},
"byStratum": {
"volatile": {
"total": 331,
"byVerdict": {
"correct": 222,
"outdated": 28,
"wrong": 10,
"unsourced": 71
},
"errors": 38,
"verifiable": 260,
"unsourced": 71,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 38,
"errorRate": 0.14615384615384616,
"errorRateWilson": {
"p": 0.14615384615384616,
"low": 0.1083692375274728,
"high": 0.1942426326249217
},
"unsourcedRate": 0.21450151057401812
},
"control": {
"total": 42,
"byVerdict": {
"correct": 37,
"outdated": 1,
"wrong": 1,
"unsourced": 3
},
"errors": 2,
"verifiable": 39,
"unsourced": 3,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 2,
"errorRate": 0.05128205128205128,
"errorRateWilson": {
"p": 0.05128205128205128,
"low": 0.014177657646399527,
"high": 0.1688593904563791
},
"unsourcedRate": 0.07142857142857142
}
},
"byClaimType": {
"version": {
"total": 33,
"byVerdict": {
"correct": 24,
"outdated": 6,
"wrong": 2,
"unsourced": 1
},
"errors": 8,
"verifiable": 32,
"unsourced": 1,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 8,
"errorRate": 0.25,
"errorRateWilson": {
"p": 0.25,
"low": 0.13252243982621553,
"high": 0.4210689177024662
},
"unsourcedRate": 0.030303030303030304
},
"tpm": {
"total": 30,
"byVerdict": {
"correct": 20,
"outdated": 4,
"wrong": 1,
"unsourced": 5
},
"errors": 5,
"verifiable": 25,
"unsourced": 5,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 5,
"errorRate": 0.2,
"errorRateWilson": {
"p": 0.2,
"low": 0.08860454100652485,
"high": 0.3913133553653825
},
"unsourcedRate": 0.16666666666666666
},
"region": {
"total": 20,
"byVerdict": {
"correct": 15,
"outdated": 2,
"wrong": 0,
"unsourced": 3
},
"errors": 2,
"verifiable": 17,
"unsourced": 3,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 2,
"errorRate": 0.11764705882352941,
"errorRateWilson": {
"p": 0.11764705882352941,
"low": 0.03287908001292092,
"high": 0.3433684249770991
},
"unsourcedRate": 0.15
},
"status": {
"total": 70,
"byVerdict": {
"correct": 58,
"outdated": 4,
"wrong": 3,
"unsourced": 5
},
"errors": 7,
"verifiable": 65,
"unsourced": 5,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 7,
"errorRate": 0.1076923076923077,
"errorRateWilson": {
"p": 0.1076923076923077,
"low": 0.05315354431925606,
"high": 0.20601533031468616
},
"unsourcedRate": 0.07142857142857142
},
"price": {
"total": 76,
"byVerdict": {
"correct": 20,
"outdated": 0,
"wrong": 0,
"unsourced": 56
},
"errors": 0,
"verifiable": 20,
"unsourced": 56,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 0,
"errorRate": 0,
"errorRateWilson": {
"p": 0,
"low": 0,
"high": 0.16113012549493322
},
"unsourcedRate": 0.7368421052631579
},
"taxonomy": {
"total": 122,
"byVerdict": {
"correct": 108,
"outdated": 7,
"wrong": 3,
"unsourced": 4
},
"errors": 10,
"verifiable": 118,
"unsourced": 4,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 10,
"errorRate": 0.0847457627118644,
"errorRateWilson": {
"p": 0.0847457627118644,
"low": 0.046682191106169606,
"high": 0.14899481904471476
},
"unsourcedRate": 0.03278688524590164
},
"sku": {
"total": 22,
"byVerdict": {
"correct": 14,
"outdated": 6,
"wrong": 2,
"unsourced": 0
},
"errors": 8,
"verifiable": 22,
"unsourced": 0,
"errorsStalenessCatchable": 0,
"errorsJudgeUnique": 8,
"errorRate": 0.36363636363636365,
"errorRateWilson": {
"p": 0.36363636363636365,
"low": 0.19732972772607899,
"high": 0.5704865065628195
},
"unsourcedRate": 0
}
},
"_verdicts": [
"correct",
"outdated",
"wrong",
"unsourced"
]
}

66
scripts/kb-eval/data/base-rate-report.md Normal file
View file

@ -0,0 +1,66 @@
# Base-rate-rapport — Fase 0 (KB korrekthet)
_Generert deterministisk av `compute-base-rate.mjs` over `gold-correctness-set.json`. Tall fra testet `lib/base-rate.mjs` (15 tester). Ikke rediger for hånd — regenerer._
**Gull-sett:** 373 påstander · metode: se gold-correctness-set.json `_meta.method`
## Verdict-vokabular
- **correct** — en hentet learn.microsoft.com-side oppgir den påståtte verdien
- **outdated** — hentet kilde viser en annen, erstattet verdi (tidsdrift)
- **wrong** — hentet kilde motsier påstanden; den var aldri korrekt
- **unsourced** — ingen hentbar MS Learn-side oppgir verdien (kan ikke verifiseres)
«Reelle feil» = outdated + wrong. **unsourced er IKKE en feil** — det er den uverifiserbare massen (priser på JS-rendrede Azure-sider som en fetch-basert judge heller ikke når). Den verifiserbare feilraten ekskluderer derfor unsourced fra nevneren.
## Overall
| metrikk | verdi |
|---|---|
| Påstander totalt | 373 |
| correct / outdated / wrong / unsourced | 259 / 29 / 11 / 74 |
| Reelle feil (outdated+wrong) | 40 |
| Verifiserbare påstander (nevner) | 299 |
| **Verifiserbar feilrate** | **40/299 = 13.4%** |
| Wilson 95 % | [10.0%, 17.7%] |
| Unsourced-andel | 74/373 = 19.8% |
| Feil staleness-loopen fanger (lastmod_changed=true) | 0 |
| **Feil kun en korrekthets-judge fanger (judge-unique)** | **40** |
> **Gate-kritisk:** `errorsJudgeUnique` = reelle feil hvis siterte kilde-lastmod IKKE endret seg etter fildato — den eneste klassen en korrekthets-judge fanger som den eksisterende staleness-loopen bommer på. Staleness-recall på de reelle feilene = 0/40 = 0.0%.
## Per stratum
### Stratum
| key | n | correct | outdated | wrong | unsourced | err-rate (verifiable) | Wilson 95% | judge-unique |
|---|---|---|---|---|---|---|---|---|
| volatile | 331 | 222 | 28 | 10 | 71 | 38/260 = 14.6% | [10.8%, 19.4%] | 38 |
| control | 42 | 37 | 1 | 1 | 3 | 2/39 = 5.1% | [1.4%, 16.9%] | 2 |
## Per skill
### Skill
| key | n | correct | outdated | wrong | unsourced | err-rate (verifiable) | Wilson 95% | judge-unique |
|---|---|---|---|---|---|---|---|---|
| ms-ai-security | 96 | 65 | 7 | 1 | 23 | 8/73 = 11.0% | [5.7%, 20.2%] | 8 |
| ms-ai-engineering | 86 | 54 | 9 | 4 | 19 | 13/67 = 19.4% | [11.7%, 30.4%] | 13 |
| ms-ai-advisor | 79 | 59 | 7 | 2 | 11 | 9/68 = 13.2% | [7.1%, 23.3%] | 9 |
| ms-ai-governance | 76 | 52 | 5 | 2 | 17 | 7/59 = 11.9% | [5.9%, 22.5%] | 7 |
| ms-ai-infrastructure | 36 | 29 | 1 | 2 | 4 | 3/32 = 9.4% | [3.2%, 24.2%] | 3 |
## Per claim_type
### Claim type
| key | n | correct | outdated | wrong | unsourced | err-rate (verifiable) | Wilson 95% | judge-unique |
|---|---|---|---|---|---|---|---|---|
| taxonomy | 122 | 108 | 7 | 3 | 4 | 10/118 = 8.5% | [4.7%, 14.9%] | 10 |
| price | 76 | 20 | 0 | 0 | 56 | 0/20 = 0.0% | [0.0%, 16.1%] | 0 |
| status | 70 | 58 | 4 | 3 | 5 | 7/65 = 10.8% | [5.3%, 20.6%] | 7 |
| version | 33 | 24 | 6 | 2 | 1 | 8/32 = 25.0% | [13.3%, 42.1%] | 8 |
| tpm | 30 | 20 | 4 | 1 | 5 | 5/25 = 20.0% | [8.9%, 39.1%] | 5 |
| sku | 22 | 14 | 6 | 2 | 0 | 8/22 = 36.4% | [19.7%, 57.0%] | 8 |
| region | 20 | 15 | 2 | 0 | 3 | 2/17 = 11.8% | [3.3%, 34.3%] | 2 |