diff --git a/docs/ref-kb-workflow-plan-2026-06.md b/docs/ref-kb-workflow-plan-2026-06.md index 32700cd..aacfd8d 100644 --- a/docs/ref-kb-workflow-plan-2026-06.md +++ b/docs/ref-kb-workflow-plan-2026-06.md @@ -150,6 +150,8 @@ Fase 0 ✅ lukket; gaten sa **BYGG Fase 3 (scoped)**. Gjenstående arbeid har ** > **FORHÅNDSREGISTRERT GATE (låst 2026-06-26, operatørvalg, FØR fan-out):** judge **recall ≥ 0,80 OG presisjon ≥ 0,70** (punktestimat), målt på den verifiserbare evaluerings-populasjonen P = volatil + fetchbar claim_type (price ekskl.) = **240 påstander, 38 positive**. PLUSS nødvendig betingelse: judge-recall > staleness-recall (staleness = 0/38). Wilson 95 %-bånd rapporteres som kontekst (n=38 → bredt bånd); grensetilfeller flagges for operatør, ikke mekanisk avvist. Strengt nivå valgt: bygg S3 KUN hvis judgen er svært sterk. **Harness (testet, 14 tester):** `lib/judge-bakeoff.mjs` + `extract-judge-claims.mjs` (blind manifest, 0 label-lekkasje) + `judge-claim-prompt.md` (blind per-påstands-judge) + `run-judge-bakeoff.mjs --min-recall 0.80 --min-precision 0.70`. Blindhet: judgen ser aldri gull-verdict; join på `id` i koden etterpå. +> **S1-RESULTAT (2026-06-26, blind fan-out 15 batcher Opus xhigh, 255 påstander dekket): GATE ❌ FAIL — recall 68,4 % < 0,80.** Rapport: `scripts/kb-eval/data/judge-bakeoff-report.{json,md}`; resultater: `judge-bakeoff-results.json`. **Judge: presisjon 81,3 % (✅ ≥0,70), recall 68,4 % (❌ <0,80, Wilson [52,5–80,9 %]), fanget 26/38, slår staleness 0/38 desisivt.** Recall-draget er **konsentrert**: sku 37,5 % (3/8) + taxonomy 66,7 % + region 50 % (n=2); version/status/tpm er allerede 80–86 % recall ved 75–100 % presisjon. 10 av 12 bommer var «grounded»-men-feil (brittle claim-dekomponering, isolert til sku/taxonomy). Per pre-registrert gate ⇒ **STOPP, bygg IKKE S3.** **ÅPEN FORK (operatør):** (a) honorer FAIL — fall tilbake på staleness + operatør-gating [ren pre-registrering]; (b) scoped judge — auto-flag kun sterke typer (version/status/tpm), operatør-gate sku/taxonomy [data-støttet mellomvei]; (c) én målrettet prompt-iterasjon på sku/taxonomy + re-kjør [flagg p-hacking-risiko: frys v1 som ærlig pre-registrert utfall]. + **S2 — Fase 2: minimal type-tag (judge-uavhengig, nyttig uansett).** Klassifiser ~389 filer `reference|template|methodology|regulatory` (sidecar-manifest el. mappekonvensjon — IKKE full YAML ennå). Skiller de ~83 kildeløse legitimt (mal/metodikk: `decision-trees`, `cost-models`) fra MS-fakta-uten-kilde. NY `scripts/kb-eval/classify-ref-type.mjs` (TDD-først). Kan kjøres før/parallelt med S1 (billig). **Scope:** klassifiser advisor-filer, men MUTÉR dem ikke (Cosmo-kollisjon). **Gate:** hver fil har én type; kildeløse delt i to bøtter; reproduserbar; suite grønn. **S3 — Fase 3-forutsetning (a)+(b): backfill + full frontmatter (KUN hvis S1 passerte).** Produksjons-scaffolding for judgen: gi hver verifiserbar fil (~306 som siterer ≥1 MS Learn-URL, **EKSKL. advisor**) en `authority_source` + full frontmatter (`type/source/verified`, OKF-form). Advisor folder inn i **S-Cosmo**. Gated bak S1 fordi dette er judge-spesifikt og wasted hvis judgen feilet. **Gate:** hver ikke-advisor verifiserbar fil har `authority_source`; frontmatter validerer; suite grønn. diff --git a/scripts/kb-eval/data/judge-bakeoff-report.json b/scripts/kb-eval/data/judge-bakeoff-report.json new file mode 100644 index 0000000..aedbc88 --- /dev/null +++ b/scripts/kb-eval/data/judge-bakeoff-report.json @@ -0,0 +1,234 @@ +{ + "_meta": { + "source": "gold-correctness-set.json + judge-bakeoff-results.json", + "thresholds": { + "minRecall": 0.8, + "minPrecision": 0.7 + }, + "judged": 255 + }, + "population": { + "total": 255, + "verifiable": 240, + "positives": 38, + "negatives": 202, + "unsourcedInP": 15 + }, + "arms": { + "staleness": { + "tp": 0, + "fp": 0, + "fn": 38, + "tn": 202, + "positives": 38, + "negatives": 202, + "flagged": 0, + "precision": null, + "recall": 0, + "f1": null, + "recallWilson": { + "p": 0, + "low": 0, + "high": 0.09181293258383999 + }, + "precisionWilson": null + }, + "judge": { + "tp": 26, + "fp": 6, + "fn": 12, + "tn": 196, + "positives": 38, + "negatives": 202, + "flagged": 32, + "precision": 0.8125, + "recall": 0.6842105263157895, + "f1": 0.742857142857143, + "recallWilson": { + "p": 0.6842105263157895, + "low": 0.5254394356005149, + "high": 0.8091557997633335 + }, + "precisionWilson": { + "p": 0.8125, + "low": 0.6469050424507756, + "high": 0.9111057606383721 + } + }, + "hybrid": { + "tp": 26, + "fp": 6, + "fn": 12, + "tn": 196, + "positives": 38, + "negatives": 202, + "flagged": 32, + "precision": 0.8125, + "recall": 0.6842105263157895, + "f1": 0.742857142857143, + "recallWilson": { + "p": 0.6842105263157895, + "low": 0.5254394356005149, + "high": 0.8091557997633335 + }, + "precisionWilson": { + "p": 0.8125, + "low": 0.6469050424507756, + "high": 0.9111057606383721 + } + } + }, + "sourceSilent": { + "onVerifiableNegative": 0, + "onVerifiableError": 2, + "agreesWithUnsourced": 5, + "disagreesWithUnsourced": 10 + }, + "byClaimType": { + "version": { + "tp": 6, + "fp": 0, + "fn": 1, + "tn": 21, + "positives": 7, + "negatives": 21, + "flagged": 6, + "precision": 1, + "recall": 0.8571428571428571, + "f1": 0.923076923076923, + "recallWilson": { + "p": 0.8571428571428571, + "low": 0.4868654966809701, + "high": 0.9743210440510252 + }, + "precisionWilson": { + "p": 1, + "low": 0.6096569663469354, + "high": 0.9999999999999999 + } + }, + "tpm": { + "tp": 4, + "fp": 0, + "fn": 1, + "tn": 20, + "positives": 5, + "negatives": 20, + "flagged": 4, + "precision": 1, + "recall": 0.8, + "f1": 0.888888888888889, + "recallWilson": { + "p": 0.8, + "low": 0.3755282641185388, + "high": 0.9637768390302125 + }, + "precisionWilson": { + "p": 1, + "low": 0.5100999795960008, + "high": 1 + } + }, + "region": { + "tp": 1, + "fp": 0, + "fn": 1, + "tn": 13, + "positives": 2, + "negatives": 13, + "flagged": 1, + "precision": 1, + "recall": 0.5, + "f1": 0.6666666666666666, + "recallWilson": { + "p": 0.5, + "low": 0.09452865480086614, + "high": 0.9054713451991339 + }, + "precisionWilson": { + "p": 1, + "low": 0.2065432914738929, + "high": 1 + } + }, + "status": { + "tp": 6, + "fp": 2, + "fn": 1, + "tn": 44, + "positives": 7, + "negatives": 46, + "flagged": 8, + "precision": 0.75, + "recall": 0.8571428571428571, + "f1": 0.7999999999999999, + "recallWilson": { + "p": 0.8571428571428571, + "low": 0.4868654966809701, + "high": 0.9743210440510252 + }, + "precisionWilson": { + "p": 0.75, + "low": 0.40926987910258916, + "high": 0.9285223111419724 + } + }, + "taxonomy": { + "tp": 6, + "fp": 4, + "fn": 3, + "tn": 84, + "positives": 9, + "negatives": 88, + "flagged": 10, + "precision": 0.6, + "recall": 0.6666666666666666, + "f1": 0.631578947368421, + "recallWilson": { + "p": 0.6666666666666666, + "low": 0.3541973474990897, + "high": 0.8794184013172571 + }, + "precisionWilson": { + "p": 0.6, + "low": 0.3126695474501863, + "high": 0.8318224187964902 + } + }, + "sku": { + "tp": 3, + "fp": 0, + "fn": 5, + "tn": 14, + "positives": 8, + "negatives": 14, + "flagged": 3, + "precision": 1, + "recall": 0.375, + "f1": 0.5454545454545454, + "recallWilson": { + "p": 0.375, + "low": 0.13684175946152105, + "high": 0.6942621454161981 + }, + "precisionWilson": { + "p": 1, + "low": 0.43849391955098227, + "high": 1 + } + } + }, + "gate": { + "pass": false, + "recallOk": false, + "precisionOk": true, + "beatsStaleness": true, + "thresholds": { + "minRecall": 0.8, + "minPrecision": 0.7 + }, + "reasons": [ + "recall 0.684 < minRecall 0.8" + ] + } +} diff --git a/scripts/kb-eval/data/judge-bakeoff-report.md b/scripts/kb-eval/data/judge-bakeoff-report.md new file mode 100644 index 0000000..83d741e --- /dev/null +++ b/scripts/kb-eval/data/judge-bakeoff-report.md @@ -0,0 +1,54 @@ +# Judge bake-off-rapport — S1 (Fase 3 de-risk) + +_Generert deterministisk av `run-judge-bakeoff.mjs` over `gold-correctness-set.json` + `judge-bakeoff-results.json`. Tall fra testet `lib/judge-bakeoff.mjs`. Ikke rediger for hånd — regenerer._ + +**Forhåndsregistrert gate (låst FØR fan-out):** recall ≥ 0.8, presisjon ≥ 0.7, OG judge-recall > staleness-recall. + +## Evaluerings-populasjon (P) + +Volatil stratum + fetchbare claim_types (price ekskludert) — der feilene bor; unngår «invertert leverage». + +| metrikk | verdi | +|---|---| +| P totalt | 255 | +| Verifiserbare (correct/outdated/wrong) | 240 | +| Positive (reelle feil å fange) | 38 | +| Negative (correct) | 202 | +| Unsourced i P (kjørt, men utenfor P/R) | 15 | + +## Arm-sammenligning (detektering over de 240 verifiserbare) + +| arm | TP | FP | FN | TN | presisjon | recall | recall Wilson 95% | F1 | +|---|---|---|---|---|---|---|---|---| +| staleness (billig baseline) | 0 | 0 | 38 | 202 | n/a | 0.0% | [0.0%, 9.2%] | n/a | +| judge (per-påstand groundedness) | 26 | 6 | 12 | 196 | 81.3% | 68.4% | [52.5%, 80.9%] | 0.743 | +| hybrid (union) | 26 | 6 | 12 | 196 | 81.3% | 68.4% | [52.5%, 80.9%] | 0.743 | + +## Judge per claim_type (verifiserbar delmengde) + +| claim_type | positive | TP | FP | FN | presisjon | recall | +|---|---|---|---|---|---|---| +| taxonomy | 9 | 6 | 4 | 3 | 60.0% | 66.7% | +| sku | 8 | 3 | 0 | 5 | 100.0% | 37.5% | +| version | 7 | 6 | 0 | 1 | 100.0% | 85.7% | +| status | 7 | 6 | 2 | 1 | 75.0% | 85.7% | +| tpm | 5 | 4 | 0 | 1 | 100.0% | 80.0% | +| region | 2 | 1 | 0 | 1 | 100.0% | 50.0% | + +## source_silent-diagnostikk + +Judgen hentet siden men fant ikke verdien. Diagnostisk, ikke et flagg. + +| signal | antall | tolkning | +|---|---|---| +| På verifiserbar feil | 2 | judge-bom: reell feil oversett via «kan ikke verifisere» | +| På verifiserbar correct | 0 | judge reproduserte ikke et korrekt faktum mennesket fant | +| Enig med unsourced | 5 | judge reproduserer den uverifiserbare grensen (godt) | +| Uenig med unsourced | 10 | judge hevdet grunnet/ugrunnet der mennesket ikke fant kilde | + +## GATE: ❌ FAIL — stopp, ikke bygg S3 + +- recall 0.684 ≥ 0.8? **nei** +- presisjon 0.813 ≥ 0.7? **ja** +- slår staleness (recall 0.000)? **ja** +- begrunnelse: recall 0.684 < minRecall 0.8 diff --git a/scripts/kb-eval/data/judge-bakeoff-results.json b/scripts/kb-eval/data/judge-bakeoff-results.json new file mode 100644 index 0000000..d55d630 --- /dev/null +++ b/scripts/kb-eval/data/judge-bakeoff-results.json @@ -0,0 +1,1376 @@ +{ + "_meta": { + "source": "blind judge fan-out (15 batches, Opus 4.8 xhigh, microsoft_docs_fetch)", + "population": "volatile + fetchable claim_type (price excluded)", + "blind": "judges never saw gold verdict; joined by id in run-judge-bakeoff.mjs", + "note_evidence": "per-claim evidence_quote omitted to bound size; reasons + full quotes in session transcript (same pattern as gold set)", + "claim_count": 255 + }, + "results": [ + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#1", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/machine-learning/prompt-flow/migrate-prompt-flow-to-agent-framework", + "reason": "retirement date April 20, 2027 matches 2027-04-20" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#2", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/databricks/mlflow3/genai/eval-monitor/concepts/judges/", + "reason": "ten single-turn built-in judges match" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#3", + "judge_verdict": "source_silent", + "evidence_url": "https://learn.microsoft.com/azure/databricks/mlflow3/genai/eval-monitor/concepts/judges/", + "reason": "Completeness/Fluency/Equivalence not listed on any learn page" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#4", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/databricks/mlflow3/genai/eval-monitor/concepts/judges/", + "reason": "seven multi-turn conversational judges match" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#5", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/evaluation-evaluators/risk-safety-evaluators", + "reason": "violence/sexual/self-harm/hate on 0-7 scale confirmed" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#6", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/evaluation-evaluators/general-purpose-evaluators", + "reason": "Groundedness/Relevance/Coherence on 1-5 Likert confirmed" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#7", + "judge_verdict": "not_grounded", + "evidence_url": "https://learn.microsoft.com/python/api/overview/azure/ai-evaluation-readme", + "reason": "claim SDK v1.14.0 but current is 1.17.0 (superseded)" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#8", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/evaluation-evaluators/agent-evaluators", + "reason": "Intent Resolution/Task Adherence/Tool Call Accuracy listed" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#9", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/databricks/mlflow3/genai/eval-monitor/concepts/scorers", + "reason": "Databricks-hosted judges, EU models, abuse-monitoring opt-out confirmed" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#10", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/databricks/mlflow3/genai/eval-monitor/production-monitoring", + "reason": "max 20 scorers per experiment confirmed" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#1", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/what-is-foundry", + "reason": "Microsoft Foundry rebrand confirmed" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#2", + "judge_verdict": "not_grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/what-is-foundry", + "reason": "~1,900 is total catalog not the sold-by-azure subset; 11,000+ total not stated" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#3", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning", + "reason": "gpt-5 2025-08-07 400k, gpt-5.5 2026-04-24 confirmed" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#4", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "reason": "gpt-4.1 2025-04-14 1,047,576 context confirmed" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#5", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "reason": "DeepSeek-R1 163,840 context confirmed" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#6", + "judge_verdict": "not_grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "reason": "claim 131,072 but DeepSeek-R1-0528 is 163,840" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#7", + "judge_verdict": "not_grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/how-to/computer-use", + "reason": "page now gpt-5.4 with new access link; computer-use-preview/2025-03-11/8192/1024 gone" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#8", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/what-is-foundry", + "reason": "SDK Python/C#/JS/Java confirmed" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#1", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "reason": "gpt-5 2025-08-07 400k/272k/128k matches" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#2", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "reason": "gpt-5.5 2026-04-24 1,050,000/922k/Dec2025 matches" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#3", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing", + "reason": "gpt-5 4,750 / gpt-5-mini 23,750 Input TPM per PTU matches" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#4", + "judge_verdict": "not_grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing", + "reason": "claim ratio 1 but page states DeepSeek output-to-input ratio 4" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#5", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure-region-availability", + "reason": "only gpt-4o/gpt-4o-mini in norwayeast residency-bound tables" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#6", + "judge_verdict": "not_grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure-region-availability", + "reason": "gpt-5.4 also Data Zone Standard norwayeast; not only gpt-5.5" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#7", + "judge_verdict": "not_grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning", + "reason": "page says access no longer restricted; registration claim contradicted" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#8", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure-region-availability", + "reason": "o3-deep-research Global Standard westus+norwayeast only" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#9", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing", + "reason": "global/data-zone min 15 incr 5, regional 25-50 confirmed" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#1", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/reliability/reliability-api-management", + "reason": "Premium v2 AZ not multi-region; multi-region only Premium classic" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#2", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/deployment-types", + "reason": "Global/DataZone/Standard residency taxonomy matches" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#3", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/api-management/api-management-howto-deploy-multi-region", + "reason": "<10s config sync, mgmt plane+dev portal primary only" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#4", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/api-management/api-management-howto-deploy-multi-region", + "reason": "regional DNS pattern matches" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#5", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities", + "reason": "semantic caching via Managed Redis + llm-semantic-cache policies" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#6", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities", + "reason": "llm-emit-token-metric with custom dimensions confirmed" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#7", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities", + "reason": "regional gateways + circuit breaker + priority load balancing" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#8", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/api-management/api-management-howto-deploy-multi-region", + "reason": "internal VNet no auto-route, port 3443 mgmt, 1433 outbound SQL" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#9", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/api-management/api-management-howto-deploy-multi-region", + "reason": "rate-limit/llm-token-limit count separately per regional gateway" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#1", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency", + "reason": "Foundry no automatic failover/DR confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#2", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency", + "reason": "conversation history in Cosmos DB enterprise_memory confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#3", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency", + "reason": "Cosmos continuous backup 30-day PITR tier confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#4", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency", + "reason": "GZRS for Storage account prescribed" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#5", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency", + "reason": "Cosmos Service-Managed Failover switches write region" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#6", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency", + "reason": "Key Vault auto-failover confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#7", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency", + "reason": "ACR geo-replication instruction confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#8", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/what-is-foundry", + "reason": "Azure AI Studio -> Microsoft Foundry brand mapping confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#9", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/ai-foundry/openai/concepts/models", + "reason": "Global training Public Preview, no residency, Norway East+Sweden Central" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#1", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/gpt-with-vision", + "reason": "10 images per chat request confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#2", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/gpt-with-vision", + "reason": "low=512x512, high=512x512 segments confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#3", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/ai-foundry/openai/overview", + "reason": "GPT-4o low flat rate 85 tokens confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#4", + "judge_verdict": "source_silent", + "evidence_url": "", + "reason": "high-detail totals 4160/6240/6208 on no learn page (evidence_url null)" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#5", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/gpt-with-vision", + "reason": "GPT-5/4.1/4.5/4o + o-series vision-enabled match" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#6", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/multimodal-search-overview", + "reason": "Document Extraction + Content Understanding recommended; Document Layout legacy" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#7", + "judge_verdict": "not_grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/multimodal-search-overview", + "reason": "AML skill also valid; exclusivity to Azure Vision contradicted" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#8", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/multimodal-search-overview", + "reason": "Content Understanding cross-page tables/chunking/PDF-DOCX-XLSX-PPTX confirmed" + }, + { + "id": "ms-ai-governance/responsible-ai/algorithmic-accountability-auditability.md#1", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/purview/ai-microsoft-purview", + "reason": "Enterprise AI apps taxonomy (Foundry/Claude Enterprise/ChatGPT Enterprise) matches" + }, + { + "id": "ms-ai-advisor/architecture/cost-models.md#3", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/microsoft-copilot-studio/requirements-messages-management", + "reason": "agent flow enforcement blocks new runs confirmed" + }, + { + "id": "ms-ai-advisor/architecture/cost-models.md#6", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/ai-builder/credit-management", + "reason": "unused capacity doesn't carry over confirmed" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#1", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/search-region-support", + "reason": "Norway East semantic ranker blank confirmed" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#2", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/search-region-support", + "reason": "Norway East agentic retrieval blank confirmed" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#3", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/search-region-support", + "reason": "Norway East query rewrite blank confirmed" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#4", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/search-region-support", + "reason": "Norway East confidential computing yes, Sweden Central no" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#5", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/search-region-support", + "reason": "Sweden Central agentic/semantic/query-rewrite yes" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#6", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/search-region-support", + "reason": "all three regions availability zones yes (3 AZ)" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#7", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/reference/region-support", + "reason": "Norway East in Foundry projects region list" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#8", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/ai-foundry/openai/concepts/models", + "reason": "o3/gpt-4.1/gpt-4o norwayeast yes; tts swedencentral only" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#1", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/search-sku-tier", + "reason": "Serverless preview 3 regions, no SLA, no tier migration" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#2", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/search-sku-tier", + "reason": "SU and CU not interchangeable confirmed" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#3", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/search-sku-tier", + "reason": "existing services switch Basic<->S1/S2/S3 confirmed" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#4", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/search-limits-quotas-capacity", + "reason": "Basic 15GB from Apr2024, 2GB older confirmed" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#5", + "judge_verdict": "not_grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/search-limits-quotas-capacity", + "reason": "claim S1 25/S2 100/... matches only superseded pre-Apr2024 row; current 160/512/1024/2048/4096" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#6", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/search-limits-quotas-capacity", + "reason": "L1/L2 up to 12 partitions/replicas confirmed" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#7", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/vector-search-how-to-quantization", + "reason": "scalar int8 fourfold (75%) reduction confirmed" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#8", + "judge_verdict": "not_grounded", + "evidence_url": "https://learn.microsoft.com/azure/search/vector-search-how-to-quantization", + "reason": "page says up to 28x/96%; claim 96.875% (32x) contradicted" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#1", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/deployment-types", + "reason": "deployment-type taxonomy confirmed" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#2", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/api-management/backends", + "reason": "up to 30 backends in pool confirmed" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#3", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities", + "reason": "Anthropic v2/Vertex AI/Bedrock endpoints confirmed" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#4", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities", + "reason": "unified model API preview confirmed" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#5", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities", + "reason": "AI gateway in Foundry preview confirmed" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#6", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities", + "reason": "semantic caching via Managed Redis confirmed" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#7", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/api-management/backends", + "reason": "429 + Retry-After up to 1 day circuit breaker confirmed" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#8", + "judge_verdict": "not_grounded", + "evidence_url": "https://learn.microsoft.com/azure/ai-foundry/openai/concepts/models", + "reason": "Azure OpenAI models available in norwayeast; 'no Norway region' contradicted" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/genaiops-llm-specific-practices.md#1", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/machine-learning/prompt-flow/migrate-prompt-flow-to-agent-framework", + "reason": "retirement 2027-04-20 + Agent Framework migration confirmed" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/genaiops-llm-specific-practices.md#2", + "judge_verdict": "not_grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/foundry-models-overview", + "reason": "claim 1600+ but page states over 1,900 (superseded)" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#1", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/azure-monitor/reference/supported-metrics/microsoft-cognitiveservices-accounts-metrics", + "reason": "TokenTransaction = input+output confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#2", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/azure-monitor/reference/supported-metrics/microsoft-cognitiveservices-accounts-metrics", + "reason": "ProcessedPromptTokens = input confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#3", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/azure-monitor/reference/supported-metrics/microsoft-cognitiveservices-accounts-metrics", + "reason": "prompt=input completion=output taxonomy confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#4", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/concepts/use-your-data", + "reason": "On Your Data two model calls confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#5", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/concepts/use-your-data", + "reason": "gpt-35-turbo-16k token estimation table matches" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#6", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/concepts/use-your-data", + "reason": "default 5 docs, 1024 chunk confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#7", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/well-architected/cost-optimization/collect-review-cost-data", + "reason": "13mo retention, FOCUS pipeline, 90/100/110% thresholds confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#8", + "judge_verdict": "grounded", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing", + "reason": "Llama-3.3-70B 8,450 Input TPM per PTU confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/reasoning-models-o1-o3-optimization.md#1", + "judge_verdict": "grounded", + "reason": "o-series 200k/100k context confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/reasoning-models-o1-o3-optimization.md#2", + "judge_verdict": "grounded", + "reason": "unsupported params list matches" + }, + { + "id": "ms-ai-advisor/prompt-engineering/reasoning-models-o1-o3-optimization.md#3", + "judge_verdict": "grounded", + "reason": "xhigh only gpt-5.1-codex-max confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/reasoning-models-o1-o3-optimization.md#4", + "judge_verdict": "grounded", + "reason": "verbosity/preamble/allowed-tools/lark_tool confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/reasoning-models-o1-o3-optimization.md#5", + "judge_verdict": "grounded", + "reason": "gpt-5 series no concise summary confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/reasoning-models-o1-o3-optimization.md#6", + "judge_verdict": "grounded", + "reason": "o3-mini/o1 markdown re-enable confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/reasoning-models-o1-o3-optimization.md#7", + "judge_verdict": "not_grounded", + "reason": "o3-pro/gpt-5-pro/gpt-5-codex access no longer restricted; registration claim superseded" + }, + { + "id": "ms-ai-advisor/architecture/adr-template.md#1", + "judge_verdict": "not_grounded", + "reason": "Graph Connectors manage ACL permissions; zero-permission claim contradicted" + }, + { + "id": "ms-ai-advisor/architecture/adr-template.md#2", + "judge_verdict": "grounded", + "reason": "Standard no SLA vs Provisioned latency target+dedicated confirmed" + }, + { + "id": "ms-ai-advisor/architecture/adr-template.md#3", + "judge_verdict": "source_silent", + "reason": "no NOK/PTU price or 1PTU~100k tokens/hr on any learn page" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#1", + "judge_verdict": "grounded", + "reason": "regional min/incr 50 (25 mini/nano) confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#2", + "judge_verdict": "grounded", + "reason": "global/data-zone min 15 incr 5 confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#3", + "judge_verdict": "grounded", + "reason": "DeepSeek/Fireworks 100+ PTU minimum confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#4", + "judge_verdict": "grounded", + "reason": "new commitment enrollment stopped Aug 1 2024 confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#5", + "judge_verdict": "grounded", + "reason": "PTU reservation cross-model (OpenAI+DeepSeek) confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#6", + "judge_verdict": "not_grounded", + "reason": "auto-renew default OFF, not ON; claim contradicted" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#7", + "judge_verdict": "grounded", + "reason": "disconnected 1yr/web monthly/non-refundable/single-service confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#8", + "judge_verdict": "grounded", + "reason": "gpt-5.1 4750 TPM/PTU ratio 8 incr 50 confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/token-optimization-and-efficiency.md#3", + "judge_verdict": "grounded", + "reason": "1024 min + 128 increment cache confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/token-optimization-and-efficiency.md#4", + "judge_verdict": "grounded", + "reason": "24h extended retention + in_memory default confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/token-optimization-and-efficiency.md#5", + "judge_verdict": "grounded", + "reason": "5-10min clear, within 1h confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/token-optimization-and-efficiency.md#7", + "judge_verdict": "grounded", + "reason": "prompt_cache_key ~15 rpm threshold confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/real-time-reasoning-performance.md#1", + "judge_verdict": "grounded", + "reason": "realtime model versions match" + }, + { + "id": "ms-ai-advisor/prompt-engineering/real-time-reasoning-performance.md#2", + "judge_verdict": "grounded", + "reason": "gpt-realtime GA status confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/real-time-reasoning-performance.md#3", + "judge_verdict": "grounded", + "reason": "GA /openai/v1 endpoint confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/real-time-reasoning-performance.md#4", + "judge_verdict": "grounded", + "reason": "none/semantic_vad/server_vad confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/real-time-reasoning-performance.md#5", + "judge_verdict": "grounded", + "reason": "East US 2 + Sweden Central global confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/real-time-reasoning-performance.md#6", + "judge_verdict": "not_grounded", + "reason": "Realtime API now GA; 'still public preview no SLA' contradicted" + }, + { + "id": "ms-ai-advisor/prompt-engineering/real-time-reasoning-performance.md#7", + "judge_verdict": "grounded", + "reason": "GPT-4o mini PTU sizing pairs match" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#1", + "judge_verdict": "grounded", + "reason": "gpt-4o 128k context confirmed" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#2", + "judge_verdict": "grounded", + "reason": "gpt-4.1 1,047,576 context (200k+) confirmed" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#3", + "judge_verdict": "not_grounded", + "reason": "o3-mini is 200k not 128k; contradicted" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#4", + "judge_verdict": "grounded", + "reason": "o1 200k context confirmed" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#5", + "judge_verdict": "not_grounded", + "reason": "quota now Tier-based; Default/Enterprise superseded" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#6", + "judge_verdict": "grounded", + "reason": "data zone batch 500M/30M quota confirmed" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#7", + "judge_verdict": "grounded", + "reason": "File Search max_prompt_tokens 20k/50k confirmed" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/monitoring-observability-ml-systems.md#1", + "judge_verdict": "grounded", + "reason": "five built-in monitoring signals confirmed" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/monitoring-observability-ml-systems.md#2", + "judge_verdict": "grounded", + "reason": "Jensen-Shannon/chi-squared/NDCG metrics confirmed" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/monitoring-observability-ml-systems.md#3", + "judge_verdict": "grounded", + "reason": "Spark instance types E4s_v3-E64s_v3 confirmed" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/monitoring-observability-ml-systems.md#4", + "judge_verdict": "grounded", + "reason": "event type + filter key confirmed" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/monitoring-observability-ml-systems.md#5", + "judge_verdict": "grounded", + "reason": "ML namespace + tables confirmed" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/monitoring-observability-ml-systems.md#6", + "judge_verdict": "grounded", + "reason": "three alert rules aggregation/threshold match" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/monitoring-observability-ml-systems.md#7", + "judge_verdict": "grounded", + "reason": "AppTraces 90-day retention confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/multi-region-azure-openai-deployment.md#1", + "judge_verdict": "grounded", + "reason": "gpt-4o norwayeast confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/multi-region-azure-openai-deployment.md#2", + "judge_verdict": "grounded", + "reason": "Sweden Central gpt-4o/gpt-35-turbo/o1 confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/multi-region-azure-openai-deployment.md#3", + "judge_verdict": "grounded", + "reason": "older chat 1U/6RPM/1000TPM confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/multi-region-azure-openai-deployment.md#4", + "judge_verdict": "grounded", + "reason": "o1 1U/1RPM/6000TPM confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/multi-region-azure-openai-deployment.md#5", + "judge_verdict": "grounded", + "reason": "o3-mini/o1-mini/o3-pro/o3/o4-mini TPM confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/multi-region-azure-openai-deployment.md#6", + "judge_verdict": "grounded", + "reason": "30 resources/region, per-model deploy limit removed confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/multi-region-azure-openai-deployment.md#7", + "judge_verdict": "grounded", + "reason": "load balancing + session affinity + circuit breaker confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/capacity-planning-dr-configurations.md#1", + "judge_verdict": "grounded", + "reason": "ProvisionedManaged SKU name confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/capacity-planning-dr-configurations.md#2", + "judge_verdict": "grounded", + "reason": "gpt-4o 2024-08-06 version confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/capacity-planning-dr-configurations.md#3", + "judge_verdict": "source_silent", + "reason": "99.9%/99.99% per-replica SLA percentages not stated on any learn page" + }, + { + "id": "ms-ai-infrastructure/bcdr/capacity-planning-dr-configurations.md#5", + "judge_verdict": "grounded", + "reason": "quota region-specific, pre-allocate secondary confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/network-resilience-patterns-ai.md#1", + "judge_verdict": "not_grounded", + "reason": "tiers are Network/IP Protection; 'Standard' tier name superseded" + }, + { + "id": "ms-ai-infrastructure/bcdr/network-resilience-patterns-ai.md#2", + "judge_verdict": "grounded", + "reason": "CognitiveServicesManagement service tag confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/network-resilience-patterns-ai.md#3", + "judge_verdict": "grounded", + "reason": "AzureCognitiveSearch service tag confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/network-resilience-patterns-ai.md#4", + "judge_verdict": "grounded", + "reason": "exponential backoff + circuit breakers confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/service-level-documentation-dr.md#1", + "judge_verdict": "grounded", + "reason": "2024-10-21 latest GA api-version confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/service-level-documentation-dr.md#2", + "judge_verdict": "grounded", + "reason": "2024-07-01 stable Search API confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/service-level-documentation-dr.md#3", + "judge_verdict": "grounded", + "reason": "P2V3/P3V3 App Service SKUs confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/service-level-documentation-dr.md#4", + "judge_verdict": "grounded", + "reason": "AI Search single-region, customer dual-index confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/service-level-documentation-dr.md#5", + "judge_verdict": "grounded", + "reason": "Key Vault MS-managed auto failover confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/service-level-documentation-dr.md#6", + "judge_verdict": "not_grounded", + "reason": "multi-region writes can't be RPO/RTO=0; <15min RPO; contradicted" + }, + { + "id": "ms-ai-infrastructure/bcdr/service-level-documentation-dr.md#7", + "judge_verdict": "grounded", + "reason": "App Configuration geo-replication recommended confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/multi-model-strategy-costs.md#1", + "judge_verdict": "grounded", + "reason": "model router 2025-11-18 active latest confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/multi-model-strategy-costs.md#2", + "judge_verdict": "not_grounded", + "reason": "router supports ~28 models, not 'up to 18'" + }, + { + "id": "ms-ai-security/cost-optimization/multi-model-strategy-costs.md#3", + "judge_verdict": "not_grounded", + "reason": "router rate limits now tier-based; Default/Enterprise values superseded" + }, + { + "id": "ms-ai-security/cost-optimization/multi-model-strategy-costs.md#4", + "judge_verdict": "grounded", + "reason": "gpt-4.1-nano 59,400 TPM/PTU confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/multi-model-strategy-costs.md#5", + "judge_verdict": "grounded", + "reason": "gpt-5/gpt-5-mini/o4-mini TPM/PTU confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/multi-model-strategy-costs.md#6", + "judge_verdict": "grounded", + "reason": "East US 2 + Sweden Central router regions confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/multi-model-strategy-costs.md#8", + "judge_verdict": "grounded", + "reason": "Balanced 1-2% / Cost 5-6% quality band confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#4", + "judge_verdict": "grounded", + "reason": "gpt-5 4750 / gpt-4.1 3000 TPM/PTU confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#5", + "judge_verdict": "grounded", + "reason": "output-input ratio gpt-5=8 gpt-4.1=4 confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#6", + "judge_verdict": "grounded", + "reason": "Copilot Studio Basic/Standard/Premium rate tiers confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#7", + "judge_verdict": "grounded", + "reason": "four thinking levels + minimal no parallel tools confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#1", + "judge_verdict": "grounded", + "reason": "Foundry RBAC rename confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#2", + "judge_verdict": "grounded", + "reason": "Foundry Tools kind=AIServices confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#3", + "judge_verdict": "grounded", + "reason": "Owner/Contributor vs Foundry User build split confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#4", + "judge_verdict": "grounded", + "reason": "no hard limit, budgets alert-only confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#5", + "judge_verdict": "grounded", + "reason": "dynamic quota no ceiling confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#6", + "judge_verdict": "not_grounded", + "reason": "quota now Tier-based; 150K-30M range superseded" + }, + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#7", + "judge_verdict": "grounded", + "reason": "serverless 200k TPM / 1000 RPM per deployment confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/vector-storage-cost-optimization.md#1", + "judge_verdict": "grounded", + "reason": "scalar int8 fourfold confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/vector-storage-cost-optimization.md#2", + "judge_verdict": "grounded", + "reason": "binary up to 28x confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/vector-storage-cost-optimization.md#3", + "judge_verdict": "grounded", + "reason": "min 1024 dims with binary confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/vector-storage-cost-optimization.md#4", + "judge_verdict": "grounded", + "reason": "HNSW 1-20% overhead confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/vector-storage-cost-optimization.md#5", + "judge_verdict": "not_grounded", + "reason": "post-Apr2024 vector quota 5/35/150/300; claim 1/12/36/72 contradicted" + }, + { + "id": "ms-ai-security/cost-optimization/vector-storage-cost-optimization.md#6", + "judge_verdict": "grounded", + "reason": "Serverless Developer 3 regions/no SLA/no migration confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/vector-storage-cost-optimization.md#7", + "judge_verdict": "not_grounded", + "reason": "vector compression GA at 2024-07-01 not 2024-11-01; contradicted" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#1", + "judge_verdict": "grounded", + "reason": "commitment-tier features list match" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#2", + "judge_verdict": "grounded", + "reason": "commitment tier single-service confirmed" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#3", + "judge_verdict": "grounded", + "reason": "Regional/DataZone/Global provisioned types confirmed" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#5", + "judge_verdict": "not_grounded", + "reason": "terms are 1-month/1-year not 1/3-year; contradicted" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#6", + "judge_verdict": "grounded", + "reason": "Anthropic v2/Vertex/unified API/Foundry preview confirmed" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#8", + "judge_verdict": "source_silent", + "reason": "no learn page states OpenAI lacks hard-limit spend cap (negative)" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#1", + "judge_verdict": "grounded", + "reason": "Standard/Global Standard/Provisioned types confirmed" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#3", + "judge_verdict": "not_grounded", + "reason": "DALL-E 3 replaced by gpt-image; o4-mini not top reasoning; superseded" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#4", + "judge_verdict": "grounded", + "reason": "Content Understanding v1.0 GA 2025-11-01 confirmed" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#5", + "judge_verdict": "grounded", + "reason": "over 1,900 models (>100+) confirmed" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#7", + "judge_verdict": "grounded", + "reason": "Cognitive Services -> Foundry Tools rename confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/small-language-models-economics.md#1", + "judge_verdict": "grounded", + "reason": "SLM <10B / LLM >10B threshold confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/small-language-models-economics.md#2", + "judge_verdict": "grounded", + "reason": "Phi-3-small 7B/medium 14B/Phi-2 2.7B confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/small-language-models-economics.md#3", + "judge_verdict": "grounded", + "reason": "Phi-4-mini 131072/4096 confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/small-language-models-economics.md#4", + "judge_verdict": "grounded", + "reason": "Phi MIT, Falcon-7B Apache 2.0 confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/small-language-models-economics.md#5", + "judge_verdict": "grounded", + "reason": "Phi-4 sidecar frameworks confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/small-language-models-economics.md#6", + "judge_verdict": "grounded", + "reason": "KAITO Phi-4-mini GPU node pool confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/small-language-models-economics.md#9", + "judge_verdict": "grounded", + "reason": "A100/T4 region availability confirmed" + }, + { + "id": "ms-ai-engineering/rag-architecture/embedding-models-selection.md#1", + "judge_verdict": "grounded", + "reason": "ada-002 1536/3-small 1536/3-large 3072 confirmed" + }, + { + "id": "ms-ai-engineering/rag-architecture/embedding-models-selection.md#2", + "judge_verdict": "grounded", + "reason": "MRL truncation 3-large 256/3-small 512 confirmed" + }, + { + "id": "ms-ai-engineering/rag-architecture/embedding-models-selection.md#4", + "judge_verdict": "source_silent", + "reason": "multilingual-e5 dims not on any learn page" + }, + { + "id": "ms-ai-engineering/rag-architecture/embedding-models-selection.md#5", + "judge_verdict": "source_silent", + "reason": "no 'Custom Models preview' embedding fine-tuning path documented" + }, + { + "id": "ms-ai-engineering/rag-architecture/embedding-models-selection.md#6", + "judge_verdict": "not_grounded", + "reason": "Basic partition 2GB or 15GB, never 1GB; contradicted" + }, + { + "id": "ms-ai-engineering/rag-architecture/embedding-models-selection.md#7", + "judge_verdict": "grounded", + "reason": "S1 25GB (pre-Apr2024) confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/endpoint-health-and-capacity-planning.md#1", + "judge_verdict": "grounded", + "reason": "AzureOpenAIAvailabilityRate PT1M/no-export confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/endpoint-health-and-capacity-planning.md#2", + "judge_verdict": "not_grounded", + "reason": "metric names PTUUtilization/ProcessingTime/TokensGenerated wrong; actual ProvisionedUtilization/GeneratedTokens" + }, + { + "id": "ms-ai-governance/monitoring-observability/endpoint-health-and-capacity-planning.md#3", + "judge_verdict": "grounded", + "reason": "per-unit RPM/TPM ratios match" + }, + { + "id": "ms-ai-governance/monitoring-observability/endpoint-health-and-capacity-planning.md#4", + "judge_verdict": "grounded", + "reason": "dynamic quota opportunistic, never below base confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/endpoint-health-and-capacity-planning.md#5", + "judge_verdict": "grounded", + "reason": "Basic 3x3=9 SU confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/endpoint-health-and-capacity-planning.md#6", + "judge_verdict": "grounded", + "reason": "Serverless CU/hr no SLA 3 regions confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/response-quality-metrics-rag.md#1", + "judge_verdict": "grounded", + "reason": "six RAG evaluators listed" + }, + { + "id": "ms-ai-governance/monitoring-observability/response-quality-metrics-rag.md#2", + "judge_verdict": "grounded", + "reason": "Fidelity/NDCG/XDCG/MaxRelevance/Holes confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/response-quality-metrics-rag.md#3", + "judge_verdict": "grounded", + "reason": "1-5 scale, default threshold 3 confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/response-quality-metrics-rag.md#4", + "judge_verdict": "grounded", + "reason": "Groundedness Pro Content Safety binary confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/response-quality-metrics-rag.md#5", + "judge_verdict": "grounded", + "reason": "Prompt flow retirement 2027-04-20 confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/response-quality-metrics-rag.md#6", + "judge_verdict": "grounded", + "reason": "is_reasoning_model o1/o3 config confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/cost-monitoring-cost-attribution.md#1", + "judge_verdict": "grounded", + "reason": "project-level chargeback + project tag confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/cost-monitoring-cost-attribution.md#2", + "judge_verdict": "grounded", + "reason": "partner models model-name-GUID under Global confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/cost-monitoring-cost-attribution.md#3", + "judge_verdict": "grounded", + "reason": "fine-tuned training/hosting/inference billing confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/cost-monitoring-cost-attribution.md#4", + "judge_verdict": "grounded", + "reason": "15-day inactive deployment auto-delete confirmed" + }, + { + "id": "ms-ai-governance/monitoring-observability/cost-monitoring-cost-attribution.md#7", + "judge_verdict": "grounded", + "reason": "paygo-inference-input-tokens meter names confirmed" + }, + { + "id": "ms-ai-infrastructure/bcdr/cost-analysis-dr-configurations.md#3", + "judge_verdict": "grounded", + "reason": "savings plan 1/3-year all-region confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/ptu-vs-paygo-economics.md#1", + "judge_verdict": "grounded", + "reason": "three provisioned types confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/ptu-vs-paygo-economics.md#2", + "judge_verdict": "grounded", + "reason": "Standard/Priority/Provisioned/Batch categories confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/ptu-vs-paygo-economics.md#3", + "judge_verdict": "grounded", + "reason": "gpt-4o 50/15, gpt-4o-mini 25/15, DeepSeek 100 confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/ptu-vs-paygo-economics.md#4", + "judge_verdict": "grounded", + "reason": "gpt-5 4750 TPM/PTU confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/ptu-vs-paygo-economics.md#5", + "judge_verdict": "grounded", + "reason": "spillover OpenAI-only confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/ptu-vs-paygo-economics.md#6", + "judge_verdict": "grounded", + "reason": "Provisioned-managed Utilization V2 + 429 at 100% confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/ai-builder-credits-transition.md#1", + "judge_verdict": "grounded", + "reason": "EoS 2025-11-01, EoL/seeded-removal 2026-11-01 confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/ai-builder-credits-transition.md#2", + "judge_verdict": "grounded", + "reason": "seeded credits removed incl EA confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/ai-builder-credits-transition.md#3", + "judge_verdict": "grounded", + "reason": "per-license credits 500/250/5000/20000/25000 confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/ai-builder-credits-transition.md#6", + "judge_verdict": "grounded", + "reason": "Copilot Studio AI Builder uses Copilot Credits confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/ai-builder-credits-transition.md#7", + "judge_verdict": "grounded", + "reason": "new customers can't buy add-ons from 2025-11-01 confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/ai-builder-credits-transition.md#8", + "judge_verdict": "grounded", + "reason": "overage grace period, monthly reset, no carry-over confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/chain-of-thought-prompting.md#1", + "judge_verdict": "grounded", + "reason": "reasoning_effort none/minimal/low/medium/high/xhigh confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/chain-of-thought-prompting.md#2", + "judge_verdict": "grounded", + "reason": "low/medium/high all except o1-mini confirmed" + }, + { + "id": "ms-ai-advisor/prompt-engineering/chain-of-thought-prompting.md#3", + "judge_verdict": "not_grounded", + "reason": "gpt-5 series does NOT support concise; claim contradicted" + }, + { + "id": "ms-ai-governance/responsible-ai/stakeholder-communication-ai-decisions.md#1", + "judge_verdict": "grounded", + "reason": "RAI scorecard public preview PDF confirmed" + }, + { + "id": "ms-ai-governance/responsible-ai/stakeholder-communication-ai-decisions.md#3", + "judge_verdict": "grounded", + "reason": "25,000 Copilot Credits per pack/month confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/model-selection-price-performance.md#4", + "judge_verdict": "grounded", + "reason": "o4-mini 5400 / gpt-4.1 3000 TPM/PTU confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/model-selection-price-performance.md#6", + "judge_verdict": "grounded", + "reason": "GPT-5 four thinking levels confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/model-selection-price-performance.md#7", + "judge_verdict": "grounded", + "reason": "model router up to 60% cost saving confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/model-selection-price-performance.md#8", + "judge_verdict": "not_grounded", + "reason": "Model Router is preview not GA; contradicted" + }, + { + "id": "ms-ai-security/cost-optimization/model-selection-price-performance.md#9", + "judge_verdict": "not_grounded", + "reason": "default model now GPT-4.1 mini not GPT-4o-mini; superseded" + }, + { + "id": "ms-ai-security/cost-optimization/token-counting-optimization.md#1", + "judge_verdict": "grounded", + "reason": "1024 min/128 granularity/5-10min TTL confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/token-counting-optimization.md#2", + "judge_verdict": "grounded", + "reason": "24h extended cache gpt-5/gpt-4.1 confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/token-counting-optimization.md#4", + "judge_verdict": "grounded", + "reason": "On Your Data retirement 2026-10-14 confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/token-counting-optimization.md#8", + "judge_verdict": "grounded", + "reason": "gpt-35-turbo-16k token table + 20% reserve confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/rag-query-cost-reduction.md#1", + "judge_verdict": "grounded", + "reason": "On Your Data per-model token sums match" + }, + { + "id": "ms-ai-security/cost-optimization/rag-query-cost-reduction.md#2", + "judge_verdict": "grounded", + "reason": "partition storage Basic2/S1 25/S2 100/S3 200 confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/rag-query-cost-reduction.md#4", + "judge_verdict": "grounded", + "reason": "agentic retrieval GA 2026-04-01 REST + Foundry IQ confirmed" + }, + { + "id": "ms-ai-security/cost-optimization/rag-query-cost-reduction.md#7", + "judge_verdict": "grounded", + "reason": "ada-002 sole On Your Data embedding confirmed" + }, + { + "id": "ms-ai-governance/responsible-ai/responsible-ai-training-awareness.md#1", + "judge_verdict": "grounded", + "reason": "module title/9 Units/Intermediate confirmed" + }, + { + "id": "ms-ai-governance/responsible-ai/responsible-ai-training-awareness.md#2", + "judge_verdict": "not_grounded", + "reason": "AI-102 has 12-mo renewal + retire 2026-06-30; 'no expiration' contradicted" + }, + { + "id": "ms-ai-security/cost-optimization/deterministic-cost-calculation-model.md#9", + "judge_verdict": "grounded", + "reason": "messages->Copilot Credits 2025-09-01 switch confirmed" + } + ] +}