From 3e39f2df6b862a4494d31ebd3710db9f9a6fe165 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 20:10:58 +0200 Subject: [PATCH] =?UTF-8?q?feat(ms-ai-architect):=20S1=20judge=20bake-off?= =?UTF-8?q?=20harness=20+=20forh=C3=A5ndsregistrert=20gate=20(TDD)=20[skip?= =?UTF-8?q?-docs]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deterministisk de-risk-harness for Fase 3-judgen (kjøres på frosset 373-påstands gull-sett): - lib/judge-bakeoff.mjs: P-filter (volatil+fetchbar, price ekskl.), confusion-matrix for 3 armer (staleness/judge/hybrid), Wilson-bånd, forhåndsregistrert gate. 14 tester. - extract-judge-claims.mjs: blind manifest (255 påstander, 0 label-lekkasje — testet invariant). - judge-claim-prompt.md: blind per-påstands groundedness-judge (Opus xhigh, microsoft_docs_fetch). - run-judge-bakeoff.mjs: join gull+results på id, gate-rapport (.json/.md). Gate FORHÅNDSREGISTRERT (operatørvalg, før fan-out): recall ≥0.80, presisjon ≥0.70, OG slår staleness (0/38). Evalueringspop P = 240 verifiserbare, 38 positive. Suite 551/551 (538 + 13 nye). --- docs/ref-kb-workflow-plan-2026-06.md | 2 + .../kb-eval/data/judge-bakeoff-claims.json | 2051 +++++++++++++++++ scripts/kb-eval/extract-judge-claims.mjs | 51 + scripts/kb-eval/judge-claim-prompt.md | 80 + scripts/kb-eval/lib/judge-bakeoff.mjs | 206 ++ scripts/kb-eval/run-judge-bakeoff.mjs | 155 ++ tests/kb-eval/test-judge-bakeoff.test.mjs | 199 ++ 7 files changed, 2744 insertions(+) create mode 100644 scripts/kb-eval/data/judge-bakeoff-claims.json create mode 100644 scripts/kb-eval/extract-judge-claims.mjs create mode 100644 scripts/kb-eval/judge-claim-prompt.md create mode 100644 scripts/kb-eval/lib/judge-bakeoff.mjs create mode 100644 scripts/kb-eval/run-judge-bakeoff.mjs create mode 100644 tests/kb-eval/test-judge-bakeoff.test.mjs diff --git a/docs/ref-kb-workflow-plan-2026-06.md b/docs/ref-kb-workflow-plan-2026-06.md index 9f09834..32700cd 100644 --- a/docs/ref-kb-workflow-plan-2026-06.md +++ b/docs/ref-kb-workflow-plan-2026-06.md @@ -148,6 +148,8 @@ Fase 0 ✅ lukket; gaten sa **BYGG Fase 3 (scoped)**. Gjenstående arbeid har ** **S1 — Judge-prototype + bake-off på frosset gull-sett (DE-RISK, retnings-avgjørende).** Bygg en minimal per-påstands groundedness-judge (Opus 4.8 xhigh) og kjør den mot de 373 gull-påstandene (hver har allerede `evidence_url` + ordrett påstand). Mål **staleness vs judge vs hybrid** på volatil populasjon, scoped til fetchbare claim_types (`taxonomy/status/version/tpm/sku/region`; `price` ekskludert). Trenger **ingen** backfill/frontmatter — gull-settet er selvbærende. **Gate:** judge presisjon/recall mot gull ≥ avtalt terskel OG slår staleness (recall 0/40); kjent felle adressert (invertert leverage — judgen «vinner» ikke ved å auto-score stabile lav-risiko-påstander). **HVIS FAIL:** stopp — judge ikke berettiget; fall tilbake på staleness + operatør-gating; **bygg IKKE S3**. ~1 sesjon de-risker hele kritisk sti. +> **FORHÅNDSREGISTRERT GATE (låst 2026-06-26, operatørvalg, FØR fan-out):** judge **recall ≥ 0,80 OG presisjon ≥ 0,70** (punktestimat), målt på den verifiserbare evaluerings-populasjonen P = volatil + fetchbar claim_type (price ekskl.) = **240 påstander, 38 positive**. PLUSS nødvendig betingelse: judge-recall > staleness-recall (staleness = 0/38). Wilson 95 %-bånd rapporteres som kontekst (n=38 → bredt bånd); grensetilfeller flagges for operatør, ikke mekanisk avvist. Strengt nivå valgt: bygg S3 KUN hvis judgen er svært sterk. **Harness (testet, 14 tester):** `lib/judge-bakeoff.mjs` + `extract-judge-claims.mjs` (blind manifest, 0 label-lekkasje) + `judge-claim-prompt.md` (blind per-påstands-judge) + `run-judge-bakeoff.mjs --min-recall 0.80 --min-precision 0.70`. Blindhet: judgen ser aldri gull-verdict; join på `id` i koden etterpå. + **S2 — Fase 2: minimal type-tag (judge-uavhengig, nyttig uansett).** Klassifiser ~389 filer `reference|template|methodology|regulatory` (sidecar-manifest el. mappekonvensjon — IKKE full YAML ennå). Skiller de ~83 kildeløse legitimt (mal/metodikk: `decision-trees`, `cost-models`) fra MS-fakta-uten-kilde. NY `scripts/kb-eval/classify-ref-type.mjs` (TDD-først). Kan kjøres før/parallelt med S1 (billig). **Scope:** klassifiser advisor-filer, men MUTÉR dem ikke (Cosmo-kollisjon). **Gate:** hver fil har én type; kildeløse delt i to bøtter; reproduserbar; suite grønn. **S3 — Fase 3-forutsetning (a)+(b): backfill + full frontmatter (KUN hvis S1 passerte).** Produksjons-scaffolding for judgen: gi hver verifiserbar fil (~306 som siterer ≥1 MS Learn-URL, **EKSKL. advisor**) en `authority_source` + full frontmatter (`type/source/verified`, OKF-form). Advisor folder inn i **S-Cosmo**. Gated bak S1 fordi dette er judge-spesifikt og wasted hvis judgen feilet. **Gate:** hver ikke-advisor verifiserbar fil har `authority_source`; frontmatter validerer; suite grønn. diff --git a/scripts/kb-eval/data/judge-bakeoff-claims.json b/scripts/kb-eval/data/judge-bakeoff-claims.json new file mode 100644 index 0000000..a1b05ab --- /dev/null +++ b/scripts/kb-eval/data/judge-bakeoff-claims.json @@ -0,0 +1,2051 @@ +{ + "_meta": { + "source": "gold-correctness-set.json", + "population": "volatile + fetchable claim_type (price excluded)", + "blind": "gold verdict/notes/lastmod_changed/stratum withheld — no label leakage", + "claim_count": 255, + "files": 45 + }, + "claims": [ + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#1", + "file": "skills/ms-ai-advisor/references/platforms/model-catalog-2026.md", + "skill": "ms-ai-advisor", + "claim": "gpt-5 | 2025-08-07 | 400K (input 272K, output 128K) | 128K", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#2", + "file": "skills/ms-ai-advisor/references/platforms/model-catalog-2026.md", + "skill": "ms-ai-advisor", + "claim": "gpt-5.5 | 2026-04-24 | 1 050 000 (input 922K) | 128K | Treningsdata des. 2025", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#3", + "file": "skills/ms-ai-advisor/references/platforms/model-catalog-2026.md", + "skill": "ms-ai-advisor", + "claim": "gpt-5 PTU 4750 input TPM/PTU; gpt-5-mini 23750", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#4", + "file": "skills/ms-ai-advisor/references/platforms/model-catalog-2026.md", + "skill": "ms-ai-advisor", + "claim": "DeepSeek-R1/V3 input/output-ratio 1 (standard, som input)", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#5", + "file": "skills/ms-ai-advisor/references/platforms/model-catalog-2026.md", + "skill": "ms-ai-advisor", + "claim": "Eneste generative modeller Norge-resident i Norway East er gpt-4o og gpt-4o-mini", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure-region-availability" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#6", + "file": "skills/ms-ai-advisor/references/platforms/model-catalog-2026.md", + "skill": "ms-ai-advisor", + "claim": "gpt-5.5 er eneste modell med Data Zone Standard (EU-residens) i Norway East", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure-region-availability" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#7", + "file": "skills/ms-ai-advisor/references/platforms/model-catalog-2026.md", + "skill": "ms-ai-advisor", + "claim": "gpt-5/gpt-5-codex/gpt-5-pro krever registrering aka.ms/oai/gpt5access", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#8", + "file": "skills/ms-ai-advisor/references/platforms/model-catalog-2026.md", + "skill": "ms-ai-advisor", + "claim": "o3-deep-research i Norway East kun via Global Standard (Norway East + West US eneste to regioner)", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure-region-availability" + }, + { + "id": "ms-ai-advisor/platforms/model-catalog-2026.md#9", + "file": "skills/ms-ai-advisor/references/platforms/model-catalog-2026.md", + "skill": "ms-ai-advisor", + "claim": "Min deployment 15 PTU (global/data zone) / 25-50 (regional); increment 5 (global/data zone) / 25-50 (regional)", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing" + }, + { + "id": "ms-ai-advisor/architecture/cost-models.md#3", + "file": "skills/ms-ai-advisor/references/architecture/cost-models.md", + "skill": "ms-ai-advisor", + "claim": "Agent flow enforcement blokkerer nye kjøringer i stedet for å deaktivere agenten", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/microsoft-copilot-studio/requirements-messages-management" + }, + { + "id": "ms-ai-advisor/architecture/cost-models.md#6", + "file": "skills/ms-ai-advisor/references/architecture/cost-models.md", + "skill": "ms-ai-advisor", + "claim": "Ubrukte credits rulles ikke over til neste måned", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/ai-builder/credit-management" + }, + { + "id": "ms-ai-advisor/prompt-engineering/reasoning-models-o1-o3-optimization.md#1", + "file": "skills/ms-ai-advisor/references/prompt-engineering/reasoning-models-o1-o3-optimization.md", + "skill": "ms-ai-advisor", + "claim": "O3/O4/O1-serien 200K input / 100K output", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning" + }, + { + "id": "ms-ai-advisor/prompt-engineering/reasoning-models-o1-o3-optimization.md#2", + "file": "skills/ms-ai-advisor/references/prompt-engineering/reasoning-models-o1-o3-optimization.md", + "skill": "ms-ai-advisor", + "claim": "Reasoning models støtter ikke temperature/top_p/penalties/logprobs/logit_bias/max_tokens", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning" + }, + { + "id": "ms-ai-advisor/prompt-engineering/reasoning-models-o1-o3-optimization.md#3", + "file": "skills/ms-ai-advisor/references/prompt-engineering/reasoning-models-o1-o3-optimization.md", + "skill": "ms-ai-advisor", + "claim": "xhigh kun gpt-5.1-codex-max", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning" + }, + { + "id": "ms-ai-advisor/prompt-engineering/reasoning-models-o1-o3-optimization.md#4", + "file": "skills/ms-ai-advisor/references/prompt-engineering/reasoning-models-o1-o3-optimization.md", + "skill": "ms-ai-advisor", + "claim": "Nye GPT-5 features: verbosity, preamble, allowed_tools, lark_tool", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning" + }, + { + "id": "ms-ai-advisor/prompt-engineering/reasoning-models-o1-o3-optimization.md#5", + "file": "skills/ms-ai-advisor/references/prompt-engineering/reasoning-models-o1-o3-optimization.md", + "skill": "ms-ai-advisor", + "claim": "concise ikke støttet av GPT-5", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning" + }, + { + "id": "ms-ai-advisor/prompt-engineering/reasoning-models-o1-o3-optimization.md#6", + "file": "skills/ms-ai-advisor/references/prompt-engineering/reasoning-models-o1-o3-optimization.md", + "skill": "ms-ai-advisor", + "claim": "O3-mini og O1 returnerer ikke markdown som standard; Formatting re-enabled", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning" + }, + { + "id": "ms-ai-advisor/prompt-engineering/reasoning-models-o1-o3-optimization.md#7", + "file": "skills/ms-ai-advisor/references/prompt-engineering/reasoning-models-o1-o3-optimization.md", + "skill": "ms-ai-advisor", + "claim": "Limited access for o3-pro/gpt-5-pro/gpt-5-codex via aka.ms-lenker", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning" + }, + { + "id": "ms-ai-advisor/architecture/adr-template.md#1", + "file": "skills/ms-ai-advisor/references/architecture/adr-template.md", + "skill": "ms-ai-advisor", + "claim": "SharePoint Embedded/Graph Connectors: zero permission management, permissions respekteres automatisk", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/microsoft-365/copilot/extensibility/data-privacy-security" + }, + { + "id": "ms-ai-advisor/architecture/adr-template.md#2", + "file": "skills/ms-ai-advisor/references/architecture/adr-template.md", + "skill": "ms-ai-advisor", + "claim": "Standard ingen latency-garanti; PTU latency SLA + dedikert kapasitet", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/concepts/provisioned-throughput" + }, + { + "id": "ms-ai-advisor/architecture/adr-template.md#3", + "file": "skills/ms-ai-advisor/references/architecture/adr-template.md", + "skill": "ms-ai-advisor", + "claim": "PTU ~1600 NOK/PTU/måned (1 PTU ~100k tokens/time)", + "claim_type": "tpm", + "evidence_url": null + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#1", + "file": "skills/ms-ai-advisor/references/platforms/azure-ai-foundry.md", + "skill": "ms-ai-advisor", + "claim": "Plattformen heter nå Microsoft Foundry (rebrandet desember 2025)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/what-is-foundry" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#2", + "file": "skills/ms-ai-advisor/references/platforms/azure-ai-foundry.md", + "skill": "ms-ai-advisor", + "claim": "1900+ Foundry Models sold by Azure; 11000+ totalt; 40+ regioner", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/what-is-foundry" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#3", + "file": "skills/ms-ai-advisor/references/platforms/azure-ai-foundry.md", + "skill": "ms-ai-advisor", + "claim": "GPT-5-tabell GA-datoer + kontekstvinduer (gpt-5 2025-08-07 400K osv. t.o.m. gpt-5.5 2026-04-24)", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#4", + "file": "skills/ms-ai-advisor/references/platforms/azure-ai-foundry.md", + "skill": "ms-ai-advisor", + "claim": "gpt-4.1 (2025-04-14) 1 million tokens kontekst", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#5", + "file": "skills/ms-ai-advisor/references/platforms/azure-ai-foundry.md", + "skill": "ms-ai-advisor", + "claim": "DeepSeek-R1 Reasoning 163840 tokens", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#6", + "file": "skills/ms-ai-advisor/references/platforms/azure-ai-foundry.md", + "skill": "ms-ai-advisor", + "claim": "DeepSeek-R1-0528 Reasoning 131072 tokens", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#7", + "file": "skills/ms-ai-advisor/references/platforms/azure-ai-foundry.md", + "skill": "ms-ai-advisor", + "claim": "computer-use-preview (2025-03-11) via Responses API; 8192 kontekst, 1024 output; aka.ms/oai/cuaaccess", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/how-to/computer-use" + }, + { + "id": "ms-ai-advisor/platforms/azure-ai-foundry.md#8", + "file": "skills/ms-ai-advisor/references/platforms/azure-ai-foundry.md", + "skill": "ms-ai-advisor", + "claim": "SDK i Python, C#, JavaScript, Java", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/what-is-foundry" + }, + { + "id": "ms-ai-advisor/prompt-engineering/token-optimization-and-efficiency.md#3", + "file": "skills/ms-ai-advisor/references/prompt-engineering/token-optimization-and-efficiency.md", + "skill": "ms-ai-advisor", + "claim": "Prompt caching min 1024 tokens; cache-treff per 128 tokens etter første 1024", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/prompt-caching" + }, + { + "id": "ms-ai-advisor/prompt-engineering/token-optimization-and-efficiency.md#4", + "file": "skills/ms-ai-advisor/references/prompt-engineering/token-optimization-and-efficiency.md", + "skill": "ms-ai-advisor", + "claim": "Extended retention 24t for GPT-5-serien + gpt-4.1; in-memory default t.o.m. gpt-5.4", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/prompt-caching" + }, + { + "id": "ms-ai-advisor/prompt-engineering/token-optimization-and-efficiency.md#5", + "file": "skills/ms-ai-advisor/references/prompt-engineering/token-optimization-and-efficiency.md", + "skill": "ms-ai-advisor", + "claim": "In-memory cache ryddes 5-10 min inaktivitet, alltid innen 1 time", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/prompt-caching" + }, + { + "id": "ms-ai-advisor/prompt-engineering/token-optimization-and-efficiency.md#7", + "file": "skills/ms-ai-advisor/references/prompt-engineering/token-optimization-and-efficiency.md", + "skill": "ms-ai-advisor", + "claim": "prompt_cache_key for >15 req/min", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/prompt-caching" + }, + { + "id": "ms-ai-advisor/prompt-engineering/chain-of-thought-prompting.md#1", + "file": "skills/ms-ai-advisor/references/prompt-engineering/chain-of-thought-prompting.md", + "skill": "ms-ai-advisor", + "claim": "reasoning_effort superset none/minimal/low/medium/high/xhigh med per-modell-nyanser", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning" + }, + { + "id": "ms-ai-advisor/prompt-engineering/chain-of-thought-prompting.md#2", + "file": "skills/ms-ai-advisor/references/prompt-engineering/chain-of-thought-prompting.md", + "skill": "ms-ai-advisor", + "claim": "reasoning_effort low/medium/high gjelder alle reasoning models unntatt o1-mini", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning" + }, + { + "id": "ms-ai-advisor/prompt-engineering/chain-of-thought-prompting.md#3", + "file": "skills/ms-ai-advisor/references/prompt-engineering/chain-of-thought-prompting.md", + "skill": "ms-ai-advisor", + "claim": "reasoning_summary for GPT-5-serien støtter auto/concise/detailed", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#1", + "file": "skills/ms-ai-advisor/references/prompt-engineering/multimodal-prompt-design.md", + "skill": "ms-ai-advisor", + "claim": "Opptil 10 bilder per chat request", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/gpt-with-vision" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#2", + "file": "skills/ms-ai-advisor/references/prompt-engineering/multimodal-prompt-design.md", + "skill": "ms-ai-advisor", + "claim": "Detail low=512x512 lavoppløselig; high=segmentert 512x512", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/gpt-with-vision" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#3", + "file": "skills/ms-ai-advisor/references/prompt-engineering/multimodal-prompt-design.md", + "skill": "ms-ai-advisor", + "claim": "GPT-4o/GPT-4 Turbo low detail = 85 tokens", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/gpt-with-vision" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#4", + "file": "skills/ms-ai-advisor/references/prompt-engineering/multimodal-prompt-design.md", + "skill": "ms-ai-advisor", + "claim": "High detail token-tall: 1024x1024=4160; 1024x1536=6240; 1536x1024=6208; GPT-4o mini low=2833", + "claim_type": "tpm", + "evidence_url": null + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#5", + "file": "skills/ms-ai-advisor/references/prompt-engineering/multimodal-prompt-design.md", + "skill": "ms-ai-advisor", + "claim": "Supported multimodal: GPT-5/GPT-4.1/GPT-4.5/GPT-4o-serier, o-serie", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/gpt-with-vision" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#6", + "file": "skills/ms-ai-advisor/references/prompt-engineering/multimodal-prompt-design.md", + "skill": "ms-ai-advisor", + "claim": "Azure AI Search content extraction: Document Extraction/Content Understanding anbefalt; Document Layout kun eksisterende pipelines", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/search/multimodal-search-overview" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#7", + "file": "skills/ms-ai-advisor/references/prompt-engineering/multimodal-prompt-design.md", + "skill": "ms-ai-advisor", + "claim": "Image-to-vector queries kun med Azure Vision multimodal embeddings vectorizer", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/search/multimodal-search-overview" + }, + { + "id": "ms-ai-advisor/prompt-engineering/multimodal-prompt-design.md#8", + "file": "skills/ms-ai-advisor/references/prompt-engineering/multimodal-prompt-design.md", + "skill": "ms-ai-advisor", + "claim": "Content Understanding skill: cross-page tabeller, semantisk chunking, AI-bildebeskrivelser, posisjonsmetadata PDF/DOCX/XLSX/PPTX", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/search/multimodal-search-overview" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#1", + "file": "skills/ms-ai-advisor/references/architecture/regional-availability-verification.md", + "skill": "ms-ai-advisor", + "claim": "Azure AI Search Semantic ranker ikke i Norway East", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/search/search-region-support" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#2", + "file": "skills/ms-ai-advisor/references/architecture/regional-availability-verification.md", + "skill": "ms-ai-advisor", + "claim": "Azure AI Search Agentic Retrieval ikke i Norway East", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/search/search-region-support" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#3", + "file": "skills/ms-ai-advisor/references/architecture/regional-availability-verification.md", + "skill": "ms-ai-advisor", + "claim": "Azure AI Search Query Rewrite ikke i Norway East", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/search/search-region-support" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#4", + "file": "skills/ms-ai-advisor/references/architecture/regional-availability-verification.md", + "skill": "ms-ai-advisor", + "claim": "Confidential computing i Norway East, ikke Sweden Central", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/search/search-region-support" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#5", + "file": "skills/ms-ai-advisor/references/architecture/regional-availability-verification.md", + "skill": "ms-ai-advisor", + "claim": "Semantic ranker/Agentic retrieval/Query rewrite tilgjengelig i Sweden Central", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/search/search-region-support" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#6", + "file": "skills/ms-ai-advisor/references/architecture/regional-availability-verification.md", + "skill": "ms-ai-advisor", + "claim": "Availability Zones 3 AZ i Norway East, Sweden Central, West Europe", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/search/search-region-support" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#7", + "file": "skills/ms-ai-advisor/references/architecture/regional-availability-verification.md", + "skill": "ms-ai-advisor", + "claim": "Microsoft Foundry tilgjengelig i Norway East", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/foundry/reference/region-support" + }, + { + "id": "ms-ai-advisor/architecture/regional-availability-verification.md#8", + "file": "skills/ms-ai-advisor/references/architecture/regional-availability-verification.md", + "skill": "ms-ai-advisor", + "claim": "Azure OpenAI Standard-modeller (GPT-4o/o3/GPT-4.1) i Norway East; TTS kun Sweden Central", + "claim_type": "region", + "evidence_url": null + }, + { + "id": "ms-ai-advisor/prompt-engineering/real-time-reasoning-performance.md#1", + "file": "skills/ms-ai-advisor/references/prompt-engineering/real-time-reasoning-performance.md", + "skill": "ms-ai-advisor", + "claim": "Realtime-modeller (juni 2026): gpt-4o-realtime-preview, gpt-realtime 2025-08-28, gpt-realtime-mini, gpt-realtime-1.5 2026-02-23 m.fl.", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/realtime-audio" + }, + { + "id": "ms-ai-advisor/prompt-engineering/real-time-reasoning-performance.md#2", + "file": "skills/ms-ai-advisor/references/prompt-engineering/real-time-reasoning-performance.md", + "skill": "ms-ai-advisor", + "claim": "gpt-realtime og gpt-realtime-1.5 er GA; preview-suffiks kun eldre gpt-4o-realtime-preview", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/realtime-audio" + }, + { + "id": "ms-ai-advisor/prompt-engineering/real-time-reasoning-performance.md#3", + "file": "skills/ms-ai-advisor/references/prompt-engineering/real-time-reasoning-performance.md", + "skill": "ms-ai-advisor", + "claim": "Realtime API GA-stier /openai/v1/realtime/, ikke date-baserte api-version", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/realtime-audio" + }, + { + "id": "ms-ai-advisor/prompt-engineering/real-time-reasoning-performance.md#4", + "file": "skills/ms-ai-advisor/references/prompt-engineering/real-time-reasoning-performance.md", + "skill": "ms-ai-advisor", + "claim": "VAD-modes: server_vad, semantic_vad, none", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/realtime-audio" + }, + { + "id": "ms-ai-advisor/prompt-engineering/real-time-reasoning-performance.md#5", + "file": "skills/ms-ai-advisor/references/prompt-engineering/real-time-reasoning-performance.md", + "skill": "ms-ai-advisor", + "claim": "Realtime API regions: East US 2, Sweden Central (global)", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/realtime-audio" + }, + { + "id": "ms-ai-advisor/prompt-engineering/real-time-reasoning-performance.md#6", + "file": "skills/ms-ai-advisor/references/prompt-engineering/real-time-reasoning-performance.md", + "skill": "ms-ai-advisor", + "claim": "Realtime API fortsatt public preview (jan 2026); ikke SLA", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/realtime-audio" + }, + { + "id": "ms-ai-advisor/prompt-engineering/real-time-reasoning-performance.md#7", + "file": "skills/ms-ai-advisor/references/prompt-engineering/real-time-reasoning-performance.md", + "skill": "ms-ai-advisor", + "claim": "PTU-throughput-estimat (GPT-4o mini): 28500 TPM/15 PTU; 5.05M/140 PTU; 650K/30 PTU", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/latency" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#1", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-cost-optimization.md", + "skill": "ms-ai-engineering", + "claim": "Commitment Tier støtter Speech/TTS/Translation/Language/Vision OCR/Document Intelligence", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/commitment-tier" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#2", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-cost-optimization.md", + "skill": "ms-ai-engineering", + "claim": "Commitment tier kan ikke brukes med multi-service-ressurs; krever dedikert single-service", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/commitment-tier" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#3", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-cost-optimization.md", + "skill": "ms-ai-engineering", + "claim": "PTU deployment-typer: Regional/Data Zone/Global Provisioned", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/concepts/provisioned-throughput-billing" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#5", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-cost-optimization.md", + "skill": "ms-ai-engineering", + "claim": "Reservasjonsrabatt 1-års eller 3-års Azure Reservations", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/concepts/provisioned-throughput-billing" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#6", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-cost-optimization.md", + "skill": "ms-ai-engineering", + "claim": "AI gateway styrer Anthropic Messages API (v2-tiers) og Google Vertex AI, unified model API (preview), integreres i Foundry (preview)", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#8", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-cost-optimization.md", + "skill": "ms-ai-engineering", + "claim": "Azure OpenAI har ingen hard limit-funksjonalitet", + "claim_type": "status", + "evidence_url": null + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#1", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-context-windows.md", + "skill": "ms-ai-engineering", + "claim": "GPT-4o 128k context window", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#2", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-context-windows.md", + "skill": "ms-ai-engineering", + "claim": "GPT-4.1 series opp til 200k+ context window", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#3", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-context-windows.md", + "skill": "ms-ai-engineering", + "claim": "o3-mini 128k context window", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#4", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-context-windows.md", + "skill": "ms-ai-engineering", + "claim": "o1 200k context window", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#5", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-context-windows.md", + "skill": "ms-ai-engineering", + "claim": "TPM-tabell med Default tier / Enterprise tier-kolonner", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/quotas-limits" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#6", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-context-windows.md", + "skill": "ms-ai-engineering", + "claim": "Batch quota gpt-4.1/gpt-4o 500M (Enterprise), 30M (Default)", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/quotas-limits" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#7", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-context-windows.md", + "skill": "ms-ai-engineering", + "claim": "File Search: max_prompt_tokens minimum 20000 (lengre 50000)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/concepts/assistants" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#1", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-cost-optimization.md", + "skill": "ms-ai-engineering", + "claim": "Serverless (Preview) kun West Central US/Switzerland North/Japan East, uten SLA, ingen migrering", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/search/search-sku-tier" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#2", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-cost-optimization.md", + "skill": "ms-ai-engineering", + "claim": "SU og CU ikke utvekslbare", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/search/search-sku-tier" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#3", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-cost-optimization.md", + "skill": "ms-ai-engineering", + "claim": "Tier switching mellom Basic og Standard (S1/S2/S3) støttet", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/search/search-sku-tier" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#4", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-cost-optimization.md", + "skill": "ms-ai-engineering", + "claim": "Basic 15 GB per partisjon (etter april 2024; eldre 2 GB)", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/search/search-limits-quotas-capacity" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#5", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-cost-optimization.md", + "skill": "ms-ai-engineering", + "claim": "Per-partisjon storage S1 25/S2 100/S3 200/L1 1TB/L2 2TB", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/search/search-limits-quotas-capacity" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#6", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-cost-optimization.md", + "skill": "ms-ai-engineering", + "claim": "Search Units L1 1-12, L2 1-12", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/search/search-limits-quotas-capacity" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#7", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-cost-optimization.md", + "skill": "ms-ai-engineering", + "claim": "Scalar quantization 75% reduksjon (float32->int8)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/search/vector-search-how-to-quantization" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-cost-optimization.md#8", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-cost-optimization.md", + "skill": "ms-ai-engineering", + "claim": "Binary quantization 96.875% reduksjon; built-in compression opptil 92.5%", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/search/vector-search-how-to-quantization" + }, + { + "id": "ms-ai-engineering/rag-architecture/embedding-models-selection.md#1", + "file": "skills/ms-ai-engineering/references/rag-architecture/embedding-models-selection.md", + "skill": "ms-ai-engineering", + "claim": "Embeddings: ada-002 1536/8191, 3-small 1536/8191, 3-large 3072/8191", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" + }, + { + "id": "ms-ai-engineering/rag-architecture/embedding-models-selection.md#2", + "file": "skills/ms-ai-engineering/references/rag-architecture/embedding-models-selection.md", + "skill": "ms-ai-engineering", + "claim": "text-embedding-3 Matryoshka-dimensjonsreduksjon (3-large 256-1024, 3-small 512)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/search/cognitive-search-skill-azure-openai-embedding" + }, + { + "id": "ms-ai-engineering/rag-architecture/embedding-models-selection.md#4", + "file": "skills/ms-ai-engineering/references/rag-architecture/embedding-models-selection.md", + "skill": "ms-ai-engineering", + "claim": "multilingual-e5-small 384/512, e5-large 1024/512, Preview (gratis)", + "claim_type": "status", + "evidence_url": null + }, + { + "id": "ms-ai-engineering/rag-architecture/embedding-models-selection.md#5", + "file": "skills/ms-ai-engineering/references/rag-architecture/embedding-models-selection.md", + "skill": "ms-ai-engineering", + "claim": "Foundry støtter fine-tuning av embedding-modeller via Custom Models (preview)", + "claim_type": "status", + "evidence_url": null + }, + { + "id": "ms-ai-engineering/rag-architecture/embedding-models-selection.md#6", + "file": "skills/ms-ai-engineering/references/rag-architecture/embedding-models-selection.md", + "skill": "ms-ai-engineering", + "claim": "Azure AI Search Basic tier 1 GB storage", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/search/search-limits-quotas-capacity" + }, + { + "id": "ms-ai-engineering/rag-architecture/embedding-models-selection.md#7", + "file": "skills/ms-ai-engineering/references/rag-architecture/embedding-models-selection.md", + "skill": "ms-ai-engineering", + "claim": "Standard S1 25 GB storage", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/search/search-limits-quotas-capacity" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#1", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-vs-foundry-tools-selection.md", + "skill": "ms-ai-engineering", + "claim": "Deployment-typer: Standard, Global Standard, Provisioned (PTU)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/deployment-types" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#3", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-vs-foundry-tools-selection.md", + "skill": "ms-ai-engineering", + "claim": "Modellserie (2026-02): o4-mini topp-reasoning, o3, GPT-4o, GPT-3.5-Turbo, DALL-E 3, Whisper", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#4", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-vs-foundry-tools-selection.md", + "skill": "ms-ai-engineering", + "claim": "Content Understanding GA v1.0/2025-11-01", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/content-understanding/choosing-right-ai-tool" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#5", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-vs-foundry-tools-selection.md", + "skill": "ms-ai-engineering", + "claim": "Model Catalog 100+ modeller", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/foundry/what-is-foundry" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#7", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-vs-foundry-tools-selection.md", + "skill": "ms-ai-engineering", + "claim": "Terminologi endret fra Cognitive Services til Foundry Tools", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/content-understanding/choosing-right-ai-tool" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#1", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "skill": "ms-ai-engineering", + "claim": "Deployment-typer: Global Standard, Data Zone Standard, Regional Standard, Provisioned (PTU)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/deployment-types" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#2", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "skill": "ms-ai-engineering", + "claim": "Backend pool inntil 30 backends", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/api-management/backends" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#3", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "skill": "ms-ai-engineering", + "claim": "AI gateway: Anthropic Messages API (v2-tiers), Google Vertex AI, Amazon Bedrock", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#4", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "skill": "ms-ai-engineering", + "claim": "Unified model API (preview): ett OpenAI-kompatibelt endepunkt med format-oversettelse", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#5", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "skill": "ms-ai-engineering", + "claim": "AI gateway i Microsoft Foundry (preview)", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#6", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "skill": "ms-ai-engineering", + "claim": "Semantisk caching via Azure Managed Redis med llm-semantic-cache-policyer", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#7", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "skill": "ms-ai-engineering", + "claim": "APIM circuit breaker håndterer 429 og Retry-After (opptil 24t)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/api-management/backends" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#8", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "skill": "ms-ai-engineering", + "claim": "Ingen Azure OpenAI-region i Norge per 2026-02; nærmeste Sweden Central/West Europe", + "claim_type": "region", + "evidence_url": null + }, + { + "id": "ms-ai-engineering/mlops-genaiops/genaiops-llm-specific-practices.md#1", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/genaiops-llm-specific-practices.md", + "skill": "ms-ai-engineering", + "claim": "Prompt Flow pensjoneres 2027-04-20; migrer til Microsoft Agent Framework", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/machine-learning/prompt-flow/migrate-prompt-flow-to-agent-framework" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/genaiops-llm-specific-practices.md#2", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/genaiops-llm-specific-practices.md", + "skill": "ms-ai-engineering", + "claim": "Model Catalog 1600+ foundation models", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/foundry-models-overview" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#1", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/llm-evaluation-production.md", + "skill": "ms-ai-engineering", + "claim": "Prompt Flow pensjoneres 2027-04-20", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/machine-learning/prompt-flow/migrate-prompt-flow-to-agent-framework" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#2", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/llm-evaluation-production.md", + "skill": "ms-ai-engineering", + "claim": "MLflow 3 single-turn judges: RelevanceToQuery, RetrievalRelevance, RetrievalGroundedness, Safety, Correctness, RetrievalSufficiency, ToolCallCorrectness, ToolCallEfficiency, Guidelines, ExpectationsGuidelines", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/databricks/mlflow3/genai/eval-monitor/concepts/judges/" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#3", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/llm-evaluation-production.md", + "skill": "ms-ai-engineering", + "claim": "MLflow built-in judges inkluderer også Completeness, Fluency, Equivalence", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/databricks/mlflow3/genai/eval-monitor/concepts/judges/" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#4", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/llm-evaluation-production.md", + "skill": "ms-ai-engineering", + "claim": "Multi-turn judges: ConversationCompleteness, UserFrustration, KnowledgeRetention, ConversationalSafety, ConversationalGuidelines, ConversationalRoleAdherence, ConversationalToolCallEfficiency", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/databricks/mlflow3/genai/eval-monitor/concepts/judges/" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#5", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/llm-evaluation-production.md", + "skill": "ms-ai-engineering", + "claim": "Safety evaluators Violence/Sexual/Self-harm/Hate-Unfairness (0-7 skala)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/evaluation-evaluators/risk-safety-evaluators" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#6", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/llm-evaluation-production.md", + "skill": "ms-ai-engineering", + "claim": "Built-in LLM Judges Groundedness/Relevance/Coherence (1-5 skala)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/evaluation-evaluators/rag-evaluators" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#7", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/llm-evaluation-production.md", + "skill": "ms-ai-engineering", + "claim": "Azure AI Evaluation SDK v1.14.0", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/python/api/overview/azure/ai-evaluation-readme" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#8", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/llm-evaluation-production.md", + "skill": "ms-ai-engineering", + "claim": "Agentic evaluators: IntentResolution, TaskAdherence, ToolCallAccuracy", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/observability" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#9", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/llm-evaluation-production.md", + "skill": "ms-ai-engineering", + "claim": "Production monitoring Databricks-hosted judges; EU-hosted i EU-workspaces; Abuse Monitoring opt-out", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/databricks/mlflow3/genai/eval-monitor/concepts/scorers" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#10", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/llm-evaluation-production.md", + "skill": "ms-ai-engineering", + "claim": "Max 20 scorers per experiment i MLflow", + "claim_type": "tpm", + "evidence_url": null + }, + { + "id": "ms-ai-engineering/mlops-genaiops/monitoring-observability-ml-systems.md#1", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/monitoring-observability-ml-systems.md", + "skill": "ms-ai-engineering", + "claim": "Built-in monitoring signals: Data Drift, Prediction Drift, Data Quality, Model Performance, Feature Attribution Drift", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/machine-learning/concept-model-monitoring" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/monitoring-observability-ml-systems.md#2", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/monitoring-observability-ml-systems.md", + "skill": "ms-ai-engineering", + "claim": "Jensen-Shannon distance (numerisk); Pearson chi-squared (kategorisk); NDCG (feature attribution drift)", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/machine-learning/concept-model-monitoring" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/monitoring-observability-ml-systems.md#3", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/monitoring-observability-ml-systems.md", + "skill": "ms-ai-engineering", + "claim": "Serverless Spark compute Standard_E4s_v3 - Standard_E64s_v3", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/machine-learning/how-to-monitor-model-performance" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/monitoring-observability-ml-systems.md#4", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/monitoring-observability-ml-systems.md", + "skill": "ms-ai-engineering", + "claim": "Event type Run status changed (ikke Dataset drift detected som er v1); filter data.RunTags.azureml_modelmonitor_threshold_breached", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/machine-learning/how-to-monitor-model-performance" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/monitoring-observability-ml-systems.md#5", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/monitoring-observability-ml-systems.md", + "skill": "ms-ai-engineering", + "claim": "Namespace Machine Learning Service Workspace (KQL AmlComputeJobEvent/AmlOnlineEndpointTrafficLog)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/machine-learning/monitor-azure-machine-learning" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/monitoring-observability-ml-systems.md#6", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/monitoring-observability-ml-systems.md", + "skill": "ms-ai-engineering", + "claim": "Alert rules: Model Deploy Failed (Total>0); Quota Utilization (Avg>90%); Unusable Nodes (Total>0)", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/machine-learning/monitor-azure-machine-learning" + }, + { + "id": "ms-ai-engineering/mlops-genaiops/monitoring-observability-ml-systems.md#7", + "file": "skills/ms-ai-engineering/references/mlops-genaiops/monitoring-observability-ml-systems.md", + "skill": "ms-ai-engineering", + "claim": "Distributed training logs stdout/stderr til AppTraces (90 dager retention)", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/machine-learning/how-to-log-search" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#1", + "file": "skills/ms-ai-engineering/references/api-management/multi-region-ai-gateway-design.md", + "skill": "ms-ai-engineering", + "claim": "Premium v2 støtter AZ men ikke multi-region; multi-region kun Premium classic", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/reliability/reliability-api-management" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#2", + "file": "skills/ms-ai-engineering/references/api-management/multi-region-ai-gateway-design.md", + "skill": "ms-ai-engineering", + "claim": "Deployment-typer data residency: Standard/Provisioned region; Data Zone innen data zone; Global Standard hvilken som helst region", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/deployment-types" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#3", + "file": "skills/ms-ai-engineering/references/api-management/multi-region-ai-gateway-design.md", + "skill": "ms-ai-engineering", + "claim": "Policy-konfig synkronisert <10 sek; management plane + developer portal kun primærregion", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/api-management/api-management-howto-deploy-multi-region" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#4", + "file": "skills/ms-ai-engineering/references/api-management/multi-region-ai-gateway-design.md", + "skill": "ms-ai-engineering", + "claim": "Regional DNS https://--01.regional.azure-api.net", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/api-management/api-management-howto-deploy-multi-region" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#5", + "file": "skills/ms-ai-engineering/references/api-management/multi-region-ai-gateway-design.md", + "skill": "ms-ai-engineering", + "claim": "Semantic caching Azure Managed Redis via llm-semantic-cache-lookup/store", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#6", + "file": "skills/ms-ai-engineering/references/api-management/multi-region-ai-gateway-design.md", + "skill": "ms-ai-engineering", + "claim": "llm-emit-token-metric med regiondimensjon", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#7", + "file": "skills/ms-ai-engineering/references/api-management/multi-region-ai-gateway-design.md", + "skill": "ms-ai-engineering", + "claim": "Multi-region gateway + circuit breaker + backend pools (priority-basert) for failover", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#8", + "file": "skills/ms-ai-engineering/references/api-management/multi-region-ai-gateway-design.md", + "skill": "ms-ai-engineering", + "claim": "Internal VNet: APIM ruter ikke automatisk; Port 3443 inbound management; Port 1433 outbound SQL fra alle regioner", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/api-management/api-management-howto-deploy-multi-region" + }, + { + "id": "ms-ai-engineering/api-management/multi-region-ai-gateway-design.md#9", + "file": "skills/ms-ai-engineering/references/api-management/multi-region-ai-gateway-design.md", + "skill": "ms-ai-engineering", + "claim": "rate-limit/llm-token-limit teller separat per regional gateway", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/api-management/api-management-howto-deploy-multi-region" + }, + { + "id": "ms-ai-governance/monitoring-observability/endpoint-health-and-capacity-planning.md#1", + "file": "skills/ms-ai-governance/references/monitoring-observability/endpoint-health-and-capacity-planning.md", + "skill": "ms-ai-governance", + "claim": "AzureOpenAIAvailabilityRate metrikk PT1M, DS Export No", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/azure-monitor/reference/supported-metrics/microsoft-cognitiveservices-accounts-metrics" + }, + { + "id": "ms-ai-governance/monitoring-observability/endpoint-health-and-capacity-planning.md#2", + "file": "skills/ms-ai-governance/references/monitoring-observability/endpoint-health-and-capacity-planning.md", + "skill": "ms-ai-governance", + "claim": "Metrikker PTUUtilization, ProcessingTime, TokensGenerated", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/azure-monitor/reference/supported-metrics/microsoft-cognitiveservices-accounts-metrics" + }, + { + "id": "ms-ai-governance/monitoring-observability/endpoint-health-and-capacity-planning.md#3", + "file": "skills/ms-ai-governance/references/monitoring-observability/endpoint-health-and-capacity-planning.md", + "skill": "ms-ai-governance", + "claim": "RPM/TPM per 1 Unit Capacity: eldre chat 6/1000; o1 1/6000; o3 1/1000; o3-mini 1/10000", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/quotas-limits" + }, + { + "id": "ms-ai-governance/monitoring-observability/endpoint-health-and-capacity-planning.md#4", + "file": "skills/ms-ai-governance/references/monitoring-observability/endpoint-health-and-capacity-planning.md", + "skill": "ms-ai-governance", + "claim": "Dynamic Quota (Preview) opportunistic burst, kan ikke redusere under base", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/how-to/dynamic-quota" + }, + { + "id": "ms-ai-governance/monitoring-observability/endpoint-health-and-capacity-planning.md#5", + "file": "skills/ms-ai-governance/references/monitoring-observability/endpoint-health-and-capacity-planning.md", + "skill": "ms-ai-governance", + "claim": "Basic-tier (etter 2024-04-03) inntil 3 partisjoner x 3 replikaer (9 SU)", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/search/search-limits-quotas-capacity" + }, + { + "id": "ms-ai-governance/monitoring-observability/endpoint-health-and-capacity-planning.md#6", + "file": "skills/ms-ai-governance/references/monitoring-observability/endpoint-health-and-capacity-planning.md", + "skill": "ms-ai-governance", + "claim": "AI Search Serverless (Preview) CU/time + per-GB, kun WCUS/Switzerland North/Japan East, ingen SLA", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/search/search-limits-quotas-capacity" + }, + { + "id": "ms-ai-governance/monitoring-observability/cost-monitoring-cost-attribution.md#1", + "file": "skills/ms-ai-governance/references/monitoring-observability/cost-monitoring-cost-attribution.md", + "skill": "ms-ai-governance", + "claim": "Foundry prosjektnivå-chargeback (Preview) automatisk project-tag, kun Models sold by Azure", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/manage-costs" + }, + { + "id": "ms-ai-governance/monitoring-observability/cost-monitoring-cost-attribution.md#2", + "file": "skills/ms-ai-governance/references/monitoring-observability/cost-monitoring-cost-attribution.md", + "skill": "ms-ai-governance", + "claim": "Partner-modeller under Global resources som model-name-GUID; Global = plassering, ikke SKU", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/manage-costs" + }, + { + "id": "ms-ai-governance/monitoring-observability/cost-monitoring-cost-attribution.md#3", + "file": "skills/ms-ai-governance/references/monitoring-observability/cost-monitoring-cost-attribution.md", + "skill": "ms-ai-governance", + "claim": "Fine-tuned modeller faktureres tre veier: training/hosting/inference", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/manage-costs" + }, + { + "id": "ms-ai-governance/monitoring-observability/cost-monitoring-cost-attribution.md#4", + "file": "skills/ms-ai-governance/references/monitoring-observability/cost-monitoring-cost-attribution.md", + "skill": "ms-ai-governance", + "claim": "Deployments inaktive 15 dager slettes automatisk", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/manage-costs" + }, + { + "id": "ms-ai-governance/monitoring-observability/cost-monitoring-cost-attribution.md#7", + "file": "skills/ms-ai-governance/references/monitoring-observability/cost-monitoring-cost-attribution.md", + "skill": "ms-ai-governance", + "claim": "Marketplace-meternavn paygo-inference-input-tokens m.fl.", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/manage-costs" + }, + { + "id": "ms-ai-governance/monitoring-observability/response-quality-metrics-rag.md#1", + "file": "skills/ms-ai-governance/references/monitoring-observability/response-quality-metrics-rag.md", + "skill": "ms-ai-governance", + "claim": "RAG-evaluatorer: Groundedness, Groundedness Pro, Relevance, Response Completeness, Retrieval, Document Retrieval", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/evaluation-evaluators/rag-evaluators" + }, + { + "id": "ms-ai-governance/monitoring-observability/response-quality-metrics-rag.md#2", + "file": "skills/ms-ai-governance/references/monitoring-observability/response-quality-metrics-rag.md", + "skill": "ms-ai-governance", + "claim": "Document Retrieval-metrikker: Fidelity, NDCG, XDCG, Max Relevance, Holes", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/evaluation-evaluators/rag-evaluators" + }, + { + "id": "ms-ai-governance/monitoring-observability/response-quality-metrics-rag.md#3", + "file": "skills/ms-ai-governance/references/monitoring-observability/response-quality-metrics-rag.md", + "skill": "ms-ai-governance", + "claim": "AI-assisterte evaluatorer Likert 1-5, default threshold 3", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/evaluation-evaluators/rag-evaluators" + }, + { + "id": "ms-ai-governance/monitoring-observability/response-quality-metrics-rag.md#4", + "file": "skills/ms-ai-governance/references/monitoring-observability/response-quality-metrics-rag.md", + "skill": "ms-ai-governance", + "claim": "Groundedness Pro bruker Azure AI Content Safety, returnerer binær True/False", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/evaluation-evaluators/rag-evaluators" + }, + { + "id": "ms-ai-governance/monitoring-observability/response-quality-metrics-rag.md#5", + "file": "skills/ms-ai-governance/references/monitoring-observability/response-quality-metrics-rag.md", + "skill": "ms-ai-governance", + "claim": "Prompt Flow utfases 2027-04-20; migrer til Microsoft Agent Framework", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/machine-learning/prompt-flow/troubleshoot-guidance" + }, + { + "id": "ms-ai-governance/monitoring-observability/response-quality-metrics-rag.md#6", + "file": "skills/ms-ai-governance/references/monitoring-observability/response-quality-metrics-rag.md", + "skill": "ms-ai-governance", + "claim": "GroundednessEvaluator har is_reasoning_model=True for o-series", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/evaluation-evaluators/rag-evaluators" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#1", + "file": "skills/ms-ai-governance/references/monitoring-observability/token-usage-tracking-attribution.md", + "skill": "ms-ai-governance", + "claim": "Metrikk TokenTransaction = total token count (input+output)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/azure-monitor/reference/supported-metrics/microsoft-cognitiveservices-accounts-metrics" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#2", + "file": "skills/ms-ai-governance/references/monitoring-observability/token-usage-tracking-attribution.md", + "skill": "ms-ai-governance", + "claim": "Metrikk ProcessedPromptTokens = tokens prosessert", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/azure-monitor/reference/supported-metrics/microsoft-cognitiveservices-accounts-metrics" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#3", + "file": "skills/ms-ai-governance/references/monitoring-observability/token-usage-tracking-attribution.md", + "skill": "ms-ai-governance", + "claim": "Metrikker PromptTokens (input) og CompletionTokens (output)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/azure-monitor/reference/supported-metrics/microsoft-cognitiveservices-accounts-metrics" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#4", + "file": "skills/ms-ai-governance/references/monitoring-observability/token-usage-tracking-attribution.md", + "skill": "ms-ai-governance", + "claim": "On Your Data (RAG) to LLM-kall per forespørsel (intent + generation)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/concepts/use-your-data" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#5", + "file": "skills/ms-ai-governance/references/monitoring-observability/token-usage-tracking-attribution.md", + "skill": "ms-ai-governance", + "claim": "gpt-35-turbo-16k: Generation 4297, Intent 1366, Response 111, Intent output 25 (~5800)", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/concepts/use-your-data" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#6", + "file": "skills/ms-ai-governance/references/monitoring-observability/token-usage-tracking-attribution.md", + "skill": "ms-ai-governance", + "claim": "RAG-defaults retrieved_document_count=5, chunk_size=1024", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/concepts/use-your-data" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#7", + "file": "skills/ms-ai-governance/references/monitoring-observability/token-usage-tracking-attribution.md", + "skill": "ms-ai-governance", + "claim": "Cost Management 13 mnd retention; FOCUS-eksport ADLS->Fabric->Power BI; budget 90/100/110%, forecast 110%", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/well-architected/cost-optimization/collect-review-cost-data" + }, + { + "id": "ms-ai-governance/monitoring-observability/token-usage-tracking-attribution.md#8", + "file": "skills/ms-ai-governance/references/monitoring-observability/token-usage-tracking-attribution.md", + "skill": "ms-ai-governance", + "claim": "PTU-metrikk PTUUtilization; Input TPM per PTU = 8450 for Llama-3.3-70B", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/azure-monitor/reference/supported-metrics/microsoft-cognitiveservices-accounts-metrics" + }, + { + "id": "ms-ai-governance/responsible-ai/stakeholder-communication-ai-decisions.md#1", + "file": "skills/ms-ai-governance/references/responsible-ai/stakeholder-communication-ai-decisions.md", + "skill": "ms-ai-governance", + "claim": "Responsible AI scorecard (preview), PDF-basert rapporteringsverktøy", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/machine-learning/concept-responsible-ai-scorecard" + }, + { + "id": "ms-ai-governance/responsible-ai/stakeholder-communication-ai-decisions.md#3", + "file": "skills/ms-ai-governance/references/responsible-ai/stakeholder-communication-ai-decisions.md", + "skill": "ms-ai-governance", + "claim": "Copilot Studio inkluderer 25000 messages", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/microsoft-365/copilot/pay-as-you-go/copilot-capacity-packs" + }, + { + "id": "ms-ai-governance/responsible-ai/algorithmic-accountability-auditability.md#1", + "file": "skills/ms-ai-governance/references/responsible-ai/algorithmic-accountability-auditability.md", + "skill": "ms-ai-governance", + "claim": "Enterprise AI apps inkluderer Foundry, Entra-apper, Anthropic Claude (Enterprise), ChatGPT Enterprise; tredjeparts som Other AI apps", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/purview/ai-microsoft-purview" + }, + { + "id": "ms-ai-governance/responsible-ai/responsible-ai-training-awareness.md#1", + "file": "skills/ms-ai-governance/references/responsible-ai/responsible-ai-training-awareness.md", + "skill": "ms-ai-governance", + "claim": "Implement a Responsible Generative AI Solution in Microsoft Foundry (9 units, intermediate)", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/training/modules/responsible-ai-studio/" + }, + { + "id": "ms-ai-governance/responsible-ai/responsible-ai-training-awareness.md#2", + "file": "skills/ms-ai-governance/references/responsible-ai/responsible-ai-training-awareness.md", + "skill": "ms-ai-governance", + "claim": "AI-cert (AI-900, AI-102) har ikke formell utløpsdato", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/credentials/certifications/azure-ai-engineer" + }, + { + "id": "ms-ai-infrastructure/bcdr/multi-region-azure-openai-deployment.md#1", + "file": "skills/ms-ai-infrastructure/references/bcdr/multi-region-azure-openai-deployment.md", + "skill": "ms-ai-infrastructure", + "claim": "Norway East primær, begrenset (gpt-4o 2024-11-20)", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure-region-availability" + }, + { + "id": "ms-ai-infrastructure/bcdr/multi-region-azure-openai-deployment.md#2", + "file": "skills/ms-ai-infrastructure/references/bcdr/multi-region-azure-openai-deployment.md", + "skill": "ms-ai-infrastructure", + "claim": "Sweden Central sekundær, bred (gpt-4o, o1, gpt-35-turbo)", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure-region-availability" + }, + { + "id": "ms-ai-infrastructure/bcdr/multi-region-azure-openai-deployment.md#3", + "file": "skills/ms-ai-infrastructure/references/bcdr/multi-region-azure-openai-deployment.md", + "skill": "ms-ai-infrastructure", + "claim": "gpt-4o og eldre chat 1/6/1000 (kapasitetsenhet/RPM/TPM)", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/how-to/quota" + }, + { + "id": "ms-ai-infrastructure/bcdr/multi-region-azure-openai-deployment.md#4", + "file": "skills/ms-ai-infrastructure/references/bcdr/multi-region-azure-openai-deployment.md", + "skill": "ms-ai-infrastructure", + "claim": "o1, o1-preview 1/1/6000", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/how-to/quota" + }, + { + "id": "ms-ai-infrastructure/bcdr/multi-region-azure-openai-deployment.md#5", + "file": "skills/ms-ai-infrastructure/references/bcdr/multi-region-azure-openai-deployment.md", + "skill": "ms-ai-infrastructure", + "claim": "o3-mini/o1-mini/o3-pro 1/1/10000; o3/o4-mini 1/1/1000", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/how-to/quota" + }, + { + "id": "ms-ai-infrastructure/bcdr/multi-region-azure-openai-deployment.md#6", + "file": "skills/ms-ai-infrastructure/references/bcdr/multi-region-azure-openai-deployment.md", + "skill": "ms-ai-infrastructure", + "claim": "Maks 30 ressurser per region; ingen per-modell deployment-grense", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/how-to/quota" + }, + { + "id": "ms-ai-infrastructure/bcdr/multi-region-azure-openai-deployment.md#7", + "file": "skills/ms-ai-infrastructure/references/bcdr/multi-region-azure-openai-deployment.md", + "skill": "ms-ai-infrastructure", + "claim": "APIM lastbalansering round-robin/vektet/prioritet/session affinity; circuit breaker 429+Retry-After", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/api-management/backends" + }, + { + "id": "ms-ai-infrastructure/bcdr/capacity-planning-dr-configurations.md#1", + "file": "skills/ms-ai-infrastructure/references/bcdr/capacity-planning-dr-configurations.md", + "skill": "ms-ai-infrastructure", + "claim": "--sku-name ProvisionedManaged (PTU)", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/how-to/quota" + }, + { + "id": "ms-ai-infrastructure/bcdr/capacity-planning-dr-configurations.md#2", + "file": "skills/ms-ai-infrastructure/references/bcdr/capacity-planning-dr-configurations.md", + "skill": "ms-ai-infrastructure", + "claim": "--model-name gpt-4o --model-version 2024-08-06", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/how-to/quota" + }, + { + "id": "ms-ai-infrastructure/bcdr/capacity-planning-dr-configurations.md#3", + "file": "skills/ms-ai-infrastructure/references/bcdr/capacity-planning-dr-configurations.md", + "skill": "ms-ai-infrastructure", + "claim": "AI Search 2 vs 3 replikaer: 99.9% vs 99.99% SLA", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/reliability/reliability-ai-search" + }, + { + "id": "ms-ai-infrastructure/bcdr/capacity-planning-dr-configurations.md#5", + "file": "skills/ms-ai-infrastructure/references/bcdr/capacity-planning-dr-configurations.md", + "skill": "ms-ai-infrastructure", + "claim": "Azure OpenAI-kvoter regionsspesifikke; pre-allokér kapasitet i DR-region", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/how-to/high-availability-resiliency" + }, + { + "id": "ms-ai-infrastructure/bcdr/cost-analysis-dr-configurations.md#3", + "file": "skills/ms-ai-infrastructure/references/bcdr/cost-analysis-dr-configurations.md", + "skill": "ms-ai-infrastructure", + "claim": "Azure Savings Plans 1-år/3-år commitment, fleksibel på tvers av regioner", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/cost-management-billing/savings-plan/decide-between-savings-plan-reservation" + }, + { + "id": "ms-ai-infrastructure/bcdr/network-resilience-patterns-ai.md#1", + "file": "skills/ms-ai-infrastructure/references/bcdr/network-resilience-patterns-ai.md", + "skill": "ms-ai-infrastructure", + "claim": "Azure Front Door DDoS Protection Standard", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/ddos-protection/ddos-protection-overview" + }, + { + "id": "ms-ai-infrastructure/bcdr/network-resilience-patterns-ai.md#2", + "file": "skills/ms-ai-infrastructure/references/bcdr/network-resilience-patterns-ai.md", + "skill": "ms-ai-infrastructure", + "claim": "NSG service tag CognitiveServicesManagement", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/virtual-network/service-tags-overview" + }, + { + "id": "ms-ai-infrastructure/bcdr/network-resilience-patterns-ai.md#3", + "file": "skills/ms-ai-infrastructure/references/bcdr/network-resilience-patterns-ai.md", + "skill": "ms-ai-infrastructure", + "claim": "NSG service tag AzureCognitiveSearch", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/virtual-network/service-tags-overview" + }, + { + "id": "ms-ai-infrastructure/bcdr/network-resilience-patterns-ai.md#4", + "file": "skills/ms-ai-infrastructure/references/bcdr/network-resilience-patterns-ai.md", + "skill": "ms-ai-infrastructure", + "claim": "Circuit Breaker + Retry exponential backoff obligatorisk for alle Azure AI API-kall", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/how-to/quota" + }, + { + "id": "ms-ai-infrastructure/bcdr/service-level-documentation-dr.md#1", + "file": "skills/ms-ai-infrastructure/references/bcdr/service-level-documentation-dr.md", + "skill": "ms-ai-infrastructure", + "claim": "Azure OpenAI inference api-version=2024-10-21", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/ai-foundry/openai/api-version-lifecycle" + }, + { + "id": "ms-ai-infrastructure/bcdr/service-level-documentation-dr.md#2", + "file": "skills/ms-ai-infrastructure/references/bcdr/service-level-documentation-dr.md", + "skill": "ms-ai-infrastructure", + "claim": "Azure AI Search api-version=2024-07-01", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/rest/api/searchservice/search-service-api-versions" + }, + { + "id": "ms-ai-infrastructure/bcdr/service-level-documentation-dr.md#3", + "file": "skills/ms-ai-infrastructure/references/bcdr/service-level-documentation-dr.md", + "skill": "ms-ai-infrastructure", + "claim": "App Service plan --sku P3v3 / P2v3", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/app-service/app-service-configure-premium-v3-tier" + }, + { + "id": "ms-ai-infrastructure/bcdr/service-level-documentation-dr.md#4", + "file": "skills/ms-ai-infrastructure/references/bcdr/service-level-documentation-dr.md", + "skill": "ms-ai-infrastructure", + "claim": "Azure AI Search dual-indexing DR, ingen built-in cross-region", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/reliability/reliability-ai-search" + }, + { + "id": "ms-ai-infrastructure/bcdr/service-level-documentation-dr.md#5", + "file": "skills/ms-ai-infrastructure/references/bcdr/service-level-documentation-dr.md", + "skill": "ms-ai-infrastructure", + "claim": "Azure Key Vault MS-managed failover (RTO/RPO Auto)", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency" + }, + { + "id": "ms-ai-infrastructure/bcdr/service-level-documentation-dr.md#6", + "file": "skills/ms-ai-infrastructure/references/bcdr/service-level-documentation-dr.md", + "skill": "ms-ai-infrastructure", + "claim": "Cosmos DB multi-region writes (RTO/RPO ~0)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/cosmos-db/continuous-backup-restore-introduction" + }, + { + "id": "ms-ai-infrastructure/bcdr/service-level-documentation-dr.md#7", + "file": "skills/ms-ai-infrastructure/references/bcdr/service-level-documentation-dr.md", + "skill": "ms-ai-infrastructure", + "claim": "App Configuration geo-replication DR", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/azure-app-configuration/concept-disaster-recovery" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#1", + "file": "skills/ms-ai-infrastructure/references/bcdr/ai-foundry-disaster-recovery-planning.md", + "skill": "ms-ai-infrastructure", + "claim": "Foundry tilbyr ikke automatisk failover/DR ut av boksen (MS-dokumentert)", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#2", + "file": "skills/ms-ai-infrastructure/references/bcdr/ai-foundry-disaster-recovery-planning.md", + "skill": "ms-ai-infrastructure", + "claim": "Samtalehistorikk Cosmos DB enterprise_memory; Cosmos DB PITR", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#3", + "file": "skills/ms-ai-infrastructure/references/bcdr/ai-foundry-disaster-recovery-planning.md", + "skill": "ms-ai-infrastructure", + "claim": "Cosmos DB continuous backup --continuous-tier Continuous30Days (30-dagers PITR)", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#4", + "file": "skills/ms-ai-infrastructure/references/bcdr/ai-foundry-disaster-recovery-planning.md", + "skill": "ms-ai-infrastructure", + "claim": "Storage GZRS anbefalt for produksjon", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#5", + "file": "skills/ms-ai-infrastructure/references/bcdr/ai-foundry-disaster-recovery-planning.md", + "skill": "ms-ai-infrastructure", + "claim": "Cosmos DB Service-Managed Failover bytter skriveregion automatisk", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#6", + "file": "skills/ms-ai-infrastructure/references/bcdr/ai-foundry-disaster-recovery-planning.md", + "skill": "ms-ai-infrastructure", + "claim": "Azure Key Vault auto-failover til sekundær", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#7", + "file": "skills/ms-ai-infrastructure/references/bcdr/ai-foundry-disaster-recovery-planning.md", + "skill": "ms-ai-infrastructure", + "claim": "Azure Container Registry geo-replikering (konfigurer)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/high-availability-resiliency" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#8", + "file": "skills/ms-ai-infrastructure/references/bcdr/ai-foundry-disaster-recovery-planning.md", + "skill": "ms-ai-infrastructure", + "claim": "Microsoft Foundry tidligere Azure AI Studio / Azure Machine Learning", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/what-is-foundry" + }, + { + "id": "ms-ai-infrastructure/bcdr/ai-foundry-disaster-recovery-planning.md#9", + "file": "skills/ms-ai-infrastructure/references/bcdr/ai-foundry-disaster-recovery-planning.md", + "skill": "ms-ai-infrastructure", + "claim": "Global training (Public Preview) rimeligere, ingen datasuverenitet; bruk regional i Norway East/Sweden Central", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/ai-foundry/openai/concepts/models" + }, + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#4", + "file": "skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md", + "skill": "ms-ai-security", + "claim": "gpt-5 PTU 4750 input TPM/PTU; gpt-4.1 3000", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing" + }, + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#5", + "file": "skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md", + "skill": "ms-ai-security", + "claim": "gpt-4.1 1 output=4 input; gpt-5 1 output=8 input", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing" + }, + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#6", + "file": "skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md", + "skill": "ms-ai-security", + "claim": "Copilot Credits-klassifisering gpt-4.1-mini Basic, gpt-4.1 Standard, gpt-5-reasoning Premium m.fl.", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/microsoft-copilot-studio/prompt-model-settings" + }, + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#7", + "file": "skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md", + "skill": "ms-ai-security", + "claim": "GPT-5 fire tenkningsnivåer (Minimal/Low/Medium/High); parallelle verktøykall ikke ved Minimal", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/how-to/model-choice-guide" + }, + { + "id": "ms-ai-security/cost-optimization/ptu-vs-paygo-economics.md#1", + "file": "skills/ms-ai-security/references/cost-optimization/ptu-vs-paygo-economics.md", + "skill": "ms-ai-security", + "claim": "Tre provisioned deployment-typer: Global/Data Zone/Regional Provisioned", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/concepts/provisioned-throughput" + }, + { + "id": "ms-ai-security/cost-optimization/ptu-vs-paygo-economics.md#2", + "file": "skills/ms-ai-security/references/cost-optimization/ptu-vs-paygo-economics.md", + "skill": "ms-ai-security", + "claim": "Fire deployment-kategorier: Standard, Priority processing, Provisioned, Batch", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/concepts/provisioned-throughput" + }, + { + "id": "ms-ai-security/cost-optimization/ptu-vs-paygo-economics.md#3", + "file": "skills/ms-ai-security/references/cost-optimization/ptu-vs-paygo-economics.md", + "skill": "ms-ai-security", + "claim": "Min PTU GPT-4o 50/15, GPT-4o-mini 25/15, DeepSeek-R1 100 global/ingen regional", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing" + }, + { + "id": "ms-ai-security/cost-optimization/ptu-vs-paygo-economics.md#4", + "file": "skills/ms-ai-security/references/cost-optimization/ptu-vs-paygo-economics.md", + "skill": "ms-ai-security", + "claim": "GPT-5 4750 input TPM per PTU", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing" + }, + { + "id": "ms-ai-security/cost-optimization/ptu-vs-paygo-economics.md#5", + "file": "skills/ms-ai-security/references/cost-optimization/ptu-vs-paygo-economics.md", + "skill": "ms-ai-security", + "claim": "Spillover (GA) alle Azure OpenAI PTU; ikke Foundry-modeller fra andre (DeepSeek, Llama)", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/concepts/provisioned-throughput" + }, + { + "id": "ms-ai-security/cost-optimization/ptu-vs-paygo-economics.md#6", + "file": "skills/ms-ai-security/references/cost-optimization/ptu-vs-paygo-economics.md", + "skill": "ms-ai-security", + "claim": "Provisioned-Managed Utilization V2-metrikk; HTTP 429 ved 100%", + "claim_type": "taxonomy", + "evidence_url": null + }, + { + "id": "ms-ai-security/cost-optimization/deterministic-cost-calculation-model.md#9", + "file": "skills/ms-ai-security/references/cost-optimization/deterministic-cost-calculation-model.md", + "skill": "ms-ai-security", + "claim": "Fra 2025-09-01 Copilot Credits erstattet messages; antall/pack og PAYG-rate uendret", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/microsoft-copilot-studio/billing-licensing" + }, + { + "id": "ms-ai-security/cost-optimization/model-selection-price-performance.md#4", + "file": "skills/ms-ai-security/references/cost-optimization/model-selection-price-performance.md", + "skill": "ms-ai-security", + "claim": "1 PTU ~5400 input tokens/min o4-mini; ~3000 GPT-4.1", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing" + }, + { + "id": "ms-ai-security/cost-optimization/model-selection-price-performance.md#6", + "file": "skills/ms-ai-security/references/cost-optimization/model-selection-price-performance.md", + "skill": "ms-ai-security", + "claim": "GPT-5 fire reasoning-nivåer (Minimal, Low, Medium default, High)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/how-to/model-choice-guide" + }, + { + "id": "ms-ai-security/cost-optimization/model-selection-price-performance.md#7", + "file": "skills/ms-ai-security/references/cost-optimization/model-selection-price-performance.md", + "skill": "ms-ai-security", + "claim": "Model Router opptil 60% kostnadsreduksjon", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/how-to/model-choice-guide" + }, + { + "id": "ms-ai-security/cost-optimization/model-selection-price-performance.md#8", + "file": "skills/ms-ai-security/references/cost-optimization/model-selection-price-performance.md", + "skill": "ms-ai-security", + "claim": "Model Router er GA-funksjonalitet i Microsoft Foundry", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/how-to/model-choice-guide" + }, + { + "id": "ms-ai-security/cost-optimization/model-selection-price-performance.md#9", + "file": "skills/ms-ai-security/references/cost-optimization/model-selection-price-performance.md", + "skill": "ms-ai-security", + "claim": "AI Builder default GPT-4o-mini for generative (per des 2024)", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/microsoft-copilot-studio/prompt-model-settings" + }, + { + "id": "ms-ai-security/cost-optimization/multi-model-strategy-costs.md#1", + "file": "skills/ms-ai-security/references/cost-optimization/multi-model-strategy-costs.md", + "skill": "ms-ai-security", + "claim": "Model Router 2025-11-18 (GA)", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/concepts/model-router" + }, + { + "id": "ms-ai-security/cost-optimization/multi-model-strategy-costs.md#2", + "file": "skills/ms-ai-security/references/cost-optimization/multi-model-strategy-costs.md", + "skill": "ms-ai-security", + "claim": "opptil 18 underliggende modeller", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/concepts/model-router" + }, + { + "id": "ms-ai-security/cost-optimization/multi-model-strategy-costs.md#3", + "file": "skills/ms-ai-security/references/cost-optimization/multi-model-strategy-costs.md", + "skill": "ms-ai-security", + "claim": "Rate limits Default/Enterprise GlobalStandard 250/250000, Enterprise 400/400000; DataZone 150/150000", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/concepts/model-router" + }, + { + "id": "ms-ai-security/cost-optimization/multi-model-strategy-costs.md#4", + "file": "skills/ms-ai-security/references/cost-optimization/multi-model-strategy-costs.md", + "skill": "ms-ai-security", + "claim": "gpt-4.1-nano 59400 input TPM/PTU", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing" + }, + { + "id": "ms-ai-security/cost-optimization/multi-model-strategy-costs.md#5", + "file": "skills/ms-ai-security/references/cost-optimization/multi-model-strategy-costs.md", + "skill": "ms-ai-security", + "claim": "gpt-5 4750; gpt-5-mini 23750; o4-mini 5400 input TPM/PTU", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing" + }, + { + "id": "ms-ai-security/cost-optimization/multi-model-strategy-costs.md#6", + "file": "skills/ms-ai-security/references/cost-optimization/multi-model-strategy-costs.md", + "skill": "ms-ai-security", + "claim": "Model Router regional: East US 2, Sweden Central", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/concepts/model-router" + }, + { + "id": "ms-ai-security/cost-optimization/multi-model-strategy-costs.md#8", + "file": "skills/ms-ai-security/references/cost-optimization/multi-model-strategy-costs.md", + "skill": "ms-ai-security", + "claim": "Balanced default 1-2% kvalitetsrange; Cost 5-6%", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/concepts/model-router" + }, + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#1", + "file": "skills/ms-ai-security/references/cost-optimization/azure-ai-foundry-cost-governance.md", + "skill": "ms-ai-security", + "claim": "Foundry RBAC-roller omdøpt: Azure AI User->Foundry User osv.", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/rbac-foundry" + }, + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#2", + "file": "skills/ms-ai-security/references/cost-optimization/azure-ai-foundry-cost-governance.md", + "skill": "ms-ai-security", + "claim": "Foundry Tools -> Foundry resource (AIServices)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/manage-costs" + }, + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#3", + "file": "skills/ms-ai-security/references/cost-optimization/azure-ai-foundry-cost-governance.md", + "skill": "ms-ai-security", + "claim": "Azure Owner/Contributor kun management; build krever Foundry User/Owner", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/multi-service-resource" + }, + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#4", + "file": "skills/ms-ai-security/references/cost-optimization/azure-ai-foundry-cost-governance.md", + "skill": "ms-ai-security", + "claim": "Azure OpenAI ikke hard limit enforcement; budgets varsler ikke stopper", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry/concepts/manage-costs" + }, + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#5", + "file": "skills/ms-ai-security/references/cost-optimization/azure-ai-foundry-cost-governance.md", + "skill": "ms-ai-security", + "claim": "Dynamic Quota (Preview) ingen takgrense", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/how-to/dynamic-quota" + }, + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#6", + "file": "skills/ms-ai-security/references/cost-optimization/azure-ai-foundry-cost-governance.md", + "skill": "ms-ai-security", + "claim": "Model Quota (TPM) varierer per tier 150K-30M", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/quotas-limits" + }, + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#7", + "file": "skills/ms-ai-security/references/cost-optimization/azure-ai-foundry-cost-governance.md", + "skill": "ms-ai-security", + "claim": "Serverless API Quota 200K TPM, 1K RPM per deployment", + "claim_type": "tpm", + "evidence_url": null + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#1", + "file": "skills/ms-ai-security/references/cost-optimization/reserved-capacity-planning.md", + "skill": "ms-ai-security", + "claim": "Regional Provisioned min/increment 50 (25 mini/nano)", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#2", + "file": "skills/ms-ai-security/references/cost-optimization/reserved-capacity-planning.md", + "skill": "ms-ai-security", + "claim": "Data Zone/Global Provisioned min 15, increment 5", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#3", + "file": "skills/ms-ai-security/references/cost-optimization/reserved-capacity-planning.md", + "skill": "ms-ai-security", + "claim": "Enkelte Foundry Models (DeepSeek/Fireworks) 100+ minimum", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#4", + "file": "skills/ms-ai-security/references/cost-optimization/reserved-capacity-planning.md", + "skill": "ms-ai-security", + "claim": "New enrollments STOPPED Aug 1 2024 (legacy commitment)", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/concepts/provisioned-migration" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#5", + "file": "skills/ms-ai-security/references/cost-optimization/reserved-capacity-planning.md", + "skill": "ms-ai-security", + "claim": "Fra mai 2025 PTU-reservasjoner cross-model sharing (Azure OpenAI + Foundry Models)", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/cost-management-billing/reservations/azure-openai" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#6", + "file": "skills/ms-ai-security/references/cost-optimization/reserved-capacity-planning.md", + "skill": "ms-ai-security", + "claim": "Autorenew ON som standard for nye reservasjoner (etter 2025-Q4)", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/cost-management-billing/reservations/azure-openai" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#7", + "file": "skills/ms-ai-security/references/cost-optimization/reserved-capacity-planning.md", + "skill": "ms-ai-security", + "claim": "Disconnected container 1 år; Web/Connected 1 måned; non-refunderbar; kun single-service", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/commitment-tier" + }, + { + "id": "ms-ai-security/cost-optimization/reserved-capacity-planning.md#8", + "file": "skills/ms-ai-security/references/cost-optimization/reserved-capacity-planning.md", + "skill": "ms-ai-security", + "claim": "gpt-5.1 4750 input TPM/PTU + output-ratio 8; 63 PTU -> rundet 100 regional", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/provisioned-throughput-sizing" + }, + { + "id": "ms-ai-security/cost-optimization/token-counting-optimization.md#1", + "file": "skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md", + "skill": "ms-ai-security", + "claim": "Min prompt 1024 tokens; granularitet 128; in-memory TTL 5-10 min idle, max 1 time", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/prompt-caching" + }, + { + "id": "ms-ai-security/cost-optimization/token-counting-optimization.md#2", + "file": "skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md", + "skill": "ms-ai-security", + "claim": "Extended cache opptil 24t; GPT-5-serien + gpt-4.1; default 24h nyere", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/prompt-caching" + }, + { + "id": "ms-ai-security/cost-optimization/token-counting-optimization.md#4", + "file": "skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md", + "skill": "ms-ai-security", + "claim": "On Your Data deprecated, retires 2026-10-14; migrer Foundry Agent Service + Foundry IQ", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/concepts/use-your-data" + }, + { + "id": "ms-ai-security/cost-optimization/token-counting-optimization.md#8", + "file": "skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md", + "skill": "ms-ai-security", + "claim": "On Your Data token-estimering intent ~1366, generation ~4297, response ~111, intent output ~25, ~5800; 20% reservert; cap 2000", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/concepts/use-your-data" + }, + { + "id": "ms-ai-security/cost-optimization/vector-storage-cost-optimization.md#1", + "file": "skills/ms-ai-security/references/cost-optimization/vector-storage-cost-optimization.md", + "skill": "ms-ai-security", + "claim": "Scalar (int8) float32->int8 4x reduksjon", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/search/vector-search-how-to-quantization" + }, + { + "id": "ms-ai-security/cost-optimization/vector-storage-cost-optimization.md#2", + "file": "skills/ms-ai-security/references/cost-optimization/vector-storage-cost-optimization.md", + "skill": "ms-ai-security", + "claim": "Binary float32->1 bit, opptil 28x reduksjon", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/search/vector-search-how-to-quantization" + }, + { + "id": "ms-ai-security/cost-optimization/vector-storage-cost-optimization.md#3", + "file": "skills/ms-ai-security/references/cost-optimization/vector-storage-cost-optimization.md", + "skill": "ms-ai-security", + "claim": "Anbefalt minstegrense 1024 dimensjoner ved binary quantization", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/search/vector-search-how-to-truncate-dimensions" + }, + { + "id": "ms-ai-security/cost-optimization/vector-storage-cost-optimization.md#4", + "file": "skills/ms-ai-security/references/cost-optimization/vector-storage-cost-optimization.md", + "skill": "ms-ai-security", + "claim": "HNSW overhead 1-20% av raw vector size", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/search/vector-search-index-size" + }, + { + "id": "ms-ai-security/cost-optimization/vector-storage-cost-optimization.md#5", + "file": "skills/ms-ai-security/references/cost-optimization/vector-storage-cost-optimization.md", + "skill": "ms-ai-security", + "claim": "Vector quota (post-April 2024) Basic 1/S1 12/S2 36/S3 72 GB", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/search/search-limits-quotas-capacity" + }, + { + "id": "ms-ai-security/cost-optimization/vector-storage-cost-optimization.md#6", + "file": "skills/ms-ai-security/references/cost-optimization/vector-storage-cost-optimization.md", + "skill": "ms-ai-security", + "claim": "Serverless (Preview) kun WCUS/Switzerland North/Japan East, ingen SLA, ingen migrering", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/search/search-limits-quotas-capacity" + }, + { + "id": "ms-ai-security/cost-optimization/vector-storage-cost-optimization.md#7", + "file": "skills/ms-ai-security/references/cost-optimization/vector-storage-cost-optimization.md", + "skill": "ms-ai-security", + "claim": "Vector quantization GA siden 2024-11-01", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/search/vector-search-index-size" + }, + { + "id": "ms-ai-security/cost-optimization/rag-query-cost-reduction.md#1", + "file": "skills/ms-ai-security/references/cost-optimization/rag-query-cost-reduction.md", + "skill": "ms-ai-security", + "claim": "Token consumption gpt-35-turbo-16k 5799, gpt-4-0613 5518, gpt-4-1106 5495, gpt-35-turbo-1106 6362", + "claim_type": "tpm", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/concepts/use-your-data" + }, + { + "id": "ms-ai-security/cost-optimization/rag-query-cost-reduction.md#2", + "file": "skills/ms-ai-security/references/cost-optimization/rag-query-cost-reduction.md", + "skill": "ms-ai-security", + "claim": "AI Search tier storage Basic 2/S1 25/S2 100/S3 200 GB per partisjon", + "claim_type": "sku", + "evidence_url": "https://learn.microsoft.com/azure/search/search-limits-quotas-capacity" + }, + { + "id": "ms-ai-security/cost-optimization/rag-query-cost-reduction.md#4", + "file": "skills/ms-ai-security/references/cost-optimization/rag-query-cost-reduction.md", + "skill": "ms-ai-security", + "claim": "Agentic retrieval GA i 2026-04-01 REST API; portal preview; bak Foundry IQ", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/search/agentic-retrieval-overview" + }, + { + "id": "ms-ai-security/cost-optimization/rag-query-cost-reduction.md#7", + "file": "skills/ms-ai-security/references/cost-optimization/rag-query-cost-reduction.md", + "skill": "ms-ai-security", + "claim": "text-embedding-ada-002 eneste supported for On Your Data vector search", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/concepts/use-your-data" + }, + { + "id": "ms-ai-security/cost-optimization/ai-builder-credits-transition.md#1", + "file": "skills/ms-ai-security/references/cost-optimization/ai-builder-credits-transition.md", + "skill": "ms-ai-security", + "claim": "AI Builder capacity add-on: Salg stoppet 1. nov 2025, EOL 1. nov 2026; seeded fjernes 1. nov 2026", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/ai-builder/endofaibcredits" + }, + { + "id": "ms-ai-security/cost-optimization/ai-builder-credits-transition.md#2", + "file": "skills/ms-ai-security/references/cost-optimization/ai-builder-credits-transition.md", + "skill": "ms-ai-security", + "claim": "Seeded credits i EA fjernes også 1. nov 2026", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/ai-builder/endofaibcredits" + }, + { + "id": "ms-ai-security/cost-optimization/ai-builder-credits-transition.md#3", + "file": "skills/ms-ai-security/references/cost-optimization/ai-builder-credits-transition.md", + "skill": "ms-ai-security", + "claim": "Seeded credits per lisens: Power Apps Premium 500, per app 250, Automate Premium 5000, F&O 20000, Sustainability 25000", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/ai-builder/credit-management" + }, + { + "id": "ms-ai-security/cost-optimization/ai-builder-credits-transition.md#6", + "file": "skills/ms-ai-security/references/cost-optimization/ai-builder-credits-transition.md", + "skill": "ms-ai-security", + "claim": "AI Builder i Copilot Studio konsumerer kun Copilot Credits", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/ai-builder/administer-licensing" + }, + { + "id": "ms-ai-security/cost-optimization/ai-builder-credits-transition.md#7", + "file": "skills/ms-ai-security/references/cost-optimization/ai-builder-credits-transition.md", + "skill": "ms-ai-security", + "claim": "Nye kunder kan ikke kjøpe AI Builder add-ons fra 1. nov 2025", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/ai-builder/administer-licensing" + }, + { + "id": "ms-ai-security/cost-optimization/ai-builder-credits-transition.md#8", + "file": "skills/ms-ai-security/references/cost-optimization/ai-builder-credits-transition.md", + "skill": "ms-ai-security", + "claim": "Overage grace period (ikke fakturert), blokkerer kjøring; månedlig reset, ingen carry-over", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/ai-builder/credit-management" + }, + { + "id": "ms-ai-security/cost-optimization/small-language-models-economics.md#1", + "file": "skills/ms-ai-security/references/cost-optimization/small-language-models-economics.md", + "skill": "ms-ai-security", + "claim": "SLM under 10 mrd parametere; LLM over 10 mrd", + "claim_type": "taxonomy", + "evidence_url": "https://learn.microsoft.com/azure/aks/concepts-ai-ml-language-models" + }, + { + "id": "ms-ai-security/cost-optimization/small-language-models-economics.md#2", + "file": "skills/ms-ai-security/references/cost-optimization/small-language-models-economics.md", + "skill": "ms-ai-security", + "claim": "Phi-3-small 7B, Phi-3-medium 14B, Phi-2 2.7B", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/aks/concepts-ai-ml-language-models" + }, + { + "id": "ms-ai-security/cost-optimization/small-language-models-economics.md#3", + "file": "skills/ms-ai-security/references/cost-optimization/small-language-models-economics.md", + "skill": "ms-ai-security", + "claim": "Phi-4-mini 131072 input, 4096 output", + "claim_type": "version", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-from-partners" + }, + { + "id": "ms-ai-security/cost-optimization/small-language-models-economics.md#4", + "file": "skills/ms-ai-security/references/cost-optimization/small-language-models-economics.md", + "skill": "ms-ai-security", + "claim": "Phi-serien MIT-lisensiert; Falcon-7B Apache 2.0", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/aks/concepts-ai-ml-language-models" + }, + { + "id": "ms-ai-security/cost-optimization/small-language-models-economics.md#5", + "file": "skills/ms-ai-security/references/cost-optimization/small-language-models-economics.md", + "skill": "ms-ai-security", + "claim": "App Service Phi-4 sidecar ASP.NET Core, FastAPI, Spring Boot, Express.js", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/app-service/scenario-ai-local-small-language-model" + }, + { + "id": "ms-ai-security/cost-optimization/small-language-models-economics.md#6", + "file": "skills/ms-ai-security/references/cost-optimization/small-language-models-economics.md", + "skill": "ms-ai-security", + "claim": "KAITO støtter Phi-4-mini med auto-GPU-provisioning", + "claim_type": "status", + "evidence_url": "https://learn.microsoft.com/azure/aks/ai-toolchain-operator" + }, + { + "id": "ms-ai-security/cost-optimization/small-language-models-economics.md#9", + "file": "skills/ms-ai-security/references/cost-optimization/small-language-models-economics.md", + "skill": "ms-ai-security", + "claim": "GPU regional: West US/West US 3/Sweden Central/Australia East (A100); West Europe (T4)", + "claim_type": "region", + "evidence_url": "https://learn.microsoft.com/azure/aks/ai-toolchain-operator" + } + ] +} diff --git a/scripts/kb-eval/extract-judge-claims.mjs b/scripts/kb-eval/extract-judge-claims.mjs new file mode 100644 index 0000000..78c8858 --- /dev/null +++ b/scripts/kb-eval/extract-judge-claims.mjs @@ -0,0 +1,51 @@ +#!/usr/bin/env node +// extract-judge-claims.mjs — S1 step: emit the BLIND claim manifest the judge +// subagents are graded on. Reads the frozen gold set, filters to the eval +// population P (volatile + fetchable, price excluded) via the tested lib, strips +// every claim to the fields the judge may see (no gold verdict / notes / +// lastmod_changed / stratum — no label leakage), and writes a flat manifest. +// +// The fan-out groups these by file at dispatch time (one judge subagent per file, +// mirroring how the gold set was built). The harness later joins the judge results +// back to gold by id and runs run-judge-bakeoff.mjs. +// +// Usage: node scripts/kb-eval/extract-judge-claims.mjs [--write] +// (default: print population summary; --write persists data/judge-bakeoff-claims.json) + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { blindClaims, evalPopulation } from './lib/judge-bakeoff.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const DATA = path.join(__dirname, 'data'); + +const gold = JSON.parse(fs.readFileSync(path.join(DATA, 'gold-correctness-set.json'), 'utf8')); +const blind = blindClaims(gold.claims); + +// File -> claim count, for fan-out sizing (read-only summary). +const byFile = {}; +for (const c of blind) (byFile[c.file] ||= 0), (byFile[c.file] += 1); +const files = Object.keys(byFile).length; +const withUrl = blind.filter((c) => c.evidence_url).length; + +if (process.argv.includes('--write')) { + const out = path.join(DATA, 'judge-bakeoff-claims.json'); + const payload = { + _meta: { + source: 'gold-correctness-set.json', + population: 'volatile + fetchable claim_type (price excluded)', + blind: 'gold verdict/notes/lastmod_changed/stratum withheld — no label leakage', + claim_count: blind.length, + files, + }, + claims: blind, + }; + fs.writeFileSync(out, JSON.stringify(payload, null, 2) + '\n'); + console.log(`wrote ${out} (${blind.length} blind claims across ${files} files)`); +} else { + console.log(`eval population P: ${evalPopulation(gold.claims).length} claims`); + console.log(`blind manifest: ${blind.length} claims across ${files} files`); + console.log(`with evidence_url: ${withUrl} | without (judge falls back to docs_search): ${blind.length - withUrl}`); + console.log('(dry run — pass --write to persist judge-bakeoff-claims.json)'); +} diff --git a/scripts/kb-eval/judge-claim-prompt.md b/scripts/kb-eval/judge-claim-prompt.md new file mode 100644 index 0000000..3548134 --- /dev/null +++ b/scripts/kb-eval/judge-claim-prompt.md @@ -0,0 +1,80 @@ +# Per-claim groundedness judge — S1 bake-off (Fase 3 de-risk) + +Canonical instruction for the per-claim correctness judge. Runs as an Opus 4.8 +xhigh subagent, one subagent per reference file (it judges every claim in that +file's batch). The dispatcher fills `` and the `` batch from +`data/judge-bakeoff-claims.json` (the BLIND manifest — it carries no gold verdict). + +The judge is **blind**: it never sees the gold label. Its verdict is joined back to +the gold set by `id` in `run-judge-bakeoff.mjs` and scored as a detection task. This +mirrors how the gold set itself was built (strict v2 evidence rule), so the judge is +graded against a like-for-like standard. + +--- + +You are a correctness judge for Microsoft AI reference documentation. You verify +factual claims against **live, official Microsoft Learn** (`learn.microsoft.com`). +Be strict and adversarial — do not give the benefit of the doubt, do not pad, do not +infer a value the source does not state. + +You are judging claims extracted from ``. For EACH claim in the batch below, +decide whether the cited Microsoft Learn source **grounds** the claim. + +## The three verdicts (exhaustive, mutually exclusive) + +- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed + value(s). The page supports the claim. (Maps to gold `correct`.) +- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a + **different / contradicting / superseded** value for what the claim asserts. The + claim disagrees with the source. (Maps to gold `outdated` + `wrong`.) +- **`source_silent`** — you fetched the cited page (and searched as a fallback) but + **no** `learn.microsoft.com` page states the claimed value at all. You cannot + confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure + pages typically lands here — that is expected, not a failure. + +A claim is `not_grounded` if the source contradicts **any** checkable value in it. +It is `grounded` only if the source supports **all** checkable values. If the source +states none of them, it is `source_silent`. + +## Procedure (per claim) + +1. **Identify the volatile assertion(s)** in the claim text. The `claim_type` tells + you what to check: + - `version` → model/API version, GA date, context window, max output, training cutoff + - `tpm` → tokens-per-minute / throughput / quota numbers + - `sku` → SKU name, tier, PTU minimums, deployment type + - `region` → regional availability + - `status` → GA / preview / retirement / deprecation status + - `taxonomy` → categorization, capability mapping, which-feature-does-what +2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's + `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not + address the assertion, run `microsoft_docs_search` to find the authoritative page. +3. **Entailment check** each checkable value against the fetched text. +4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a + verbatim quote you actually fetched from a `learn.microsoft.com` URL that states + the relevant value. No quote → `source_silent`. Never quote from memory. + +## Hard rules + +- Verify against the fetched page only. Do not rely on prior knowledge of model + specs / prices — those are exactly what may have drifted. +- Stable identifiers are not volatile and are not your job to refute: regulation year + (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10 + 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on + whatever volatile value it carries, else `source_silent`. +- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence. +- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the + verdict (empty string for `source_silent`). `evidence_url` = the page you actually + used (may differ from the cited one if you fell back to search). + +## Batch to judge (from ``) + + + +## Output (strict JSON, no fence) + +``` +{"file":"","results":[ + {"id":"","judge_verdict":"grounded|not_grounded|source_silent","evidence_url":"","evidence_quote":"","reason":""} +]} +``` diff --git a/scripts/kb-eval/lib/judge-bakeoff.mjs b/scripts/kb-eval/lib/judge-bakeoff.mjs new file mode 100644 index 0000000..ece7354 --- /dev/null +++ b/scripts/kb-eval/lib/judge-bakeoff.mjs @@ -0,0 +1,206 @@ +// scripts/kb-eval/lib/judge-bakeoff.mjs — S1 judge bake-off aggregation (deterministic). +// +// De-risks the Fase 3 correctness judge BEFORE production scaffolding (S3) is built: +// grade a per-claim groundedness judge against the frozen gold set and decide whether +// it beats the cheap staleness baseline (recall 0/40) by enough to justify ~2700 +// non-batchable microsoft_docs_fetch calls per full pass. Pure functions only — no +// I/O, no Date.now/Math.random. +// +// Eval population P = volatile stratum + fetchable claim_type (price excluded). This +// is the only population that matters: it is where the real errors concentrate, and +// scoping to it structurally avoids the "inverted leverage" trap (a judge that wins +// only by auto-scoring cheap stable claims it was never at risk on). +// +// Detection confusion matrix vs the gold verdicts: +// gold positive = verdict ∈ {outdated, wrong} (a real error the judge should catch) +// gold negative = verdict === 'correct' +// excluded = verdict === 'unsourced' (no ground-truth value to grade against) +// predicted positive (flag) = judge_verdict === 'not_grounded' +// judge_verdict vocabulary: 'grounded' | 'not_grounded' | 'source_silent'. + +import { wilson } from './base-rate.mjs'; + +export const FETCHABLE_TYPES = new Set([ + 'taxonomy', 'status', 'version', 'tpm', 'sku', 'region', +]); +const ERROR_VERDICTS = new Set(['outdated', 'wrong']); + +// P = the claims the bake-off is measured on. A claim is in P iff it is volatile and +// of a fetchable claim_type (price is excluded — 74% unsourced, JS-rendered Azure +// pages defeat microsoft_docs_fetch, so a fetch-based judge cannot reach it either). +export function evalPopulation(claims) { + return claims.filter( + (c) => c.stratum === 'volatile' && FETCHABLE_TYPES.has(c.claim_type), + ); +} + +// The BLIND manifest handed to the judge subagents: every claim in P, stripped to +// the fields the judge is allowed to see. The gold verdict, notes, lastmod_changed +// and stratum are deliberately withheld so the eval is blind (no label leakage). +// All of P is included (unsourced too) — the judge must not know which claims the +// human could not source; reproducing that boundary as source_silent is itself a +// measured signal. +const BLIND_FIELDS = ['id', 'file', 'skill', 'claim', 'claim_type', 'evidence_url']; +export function blindClaims(claims) { + return evalPopulation(claims).map((c) => { + const out = {}; + for (const k of BLIND_FIELDS) out[k] = c[k]; + return out; + }); +} + +export function stalenessFlag(claim) { + return claim.lastmod_changed === true; +} +export function judgeFlag(claim) { + return claim.judge_verdict === 'not_grounded'; +} +export function hybridFlag(claim) { + return stalenessFlag(claim) || judgeFlag(claim); +} + +// Grade one detection arm over a list of *verifiable* claims (gold correct/outdated/ +// wrong only — unsourced must be filtered out before calling). flagFn(claim) -> bool +// is the arm's predicted-error rule. Returns the confusion matrix plus precision / +// recall / F1. precision is null when nothing is flagged (0/0, not NaN); recall is +// null when there are no positives at all. +export function gradeArm(verifiableClaims, flagFn) { + let tp = 0, fp = 0, fn = 0, tn = 0; + for (const c of verifiableClaims) { + const isError = ERROR_VERDICTS.has(c.verdict); + const flagged = flagFn(c); + if (isError && flagged) tp += 1; + else if (!isError && flagged) fp += 1; + else if (isError && !flagged) fn += 1; + else tn += 1; + } + const positives = tp + fn; + const negatives = fp + tn; + const flagged = tp + fp; + const precision = flagged ? tp / flagged : null; + const recall = positives ? tp / positives : null; + const f1 = + precision != null && recall != null && precision + recall > 0 + ? (2 * precision * recall) / (precision + recall) + : null; + // Wilson 95% bands surface sampling noise — the positive sample is small (≈38), + // so a point estimate near the threshold should not be read as crisp. Bands are + // reported context; the gate itself uses the point estimate. + const recallWilson = positives ? wilson(tp, positives) : null; + const precisionWilson = flagged ? wilson(tp, flagged) : null; + return { + tp, fp, fn, tn, positives, negatives, flagged, + precision, recall, f1, recallWilson, precisionWilson, + }; +} + +// The PRE-REGISTERED gate. thresholds = { minRecall, minPrecision }. Pass requires +// the judge to clear both thresholds AND strictly beat the staleness baseline's +// recall (the directional point of S1: staleness recall is 0/40, so any positive +// recall beats it — but the threshold guards against a judge too weak to justify +// the fetch cost). A null judge precision (judge flagged nothing) cannot clear a +// positive minPrecision. +export function gateDecision(judgeArm, stalenessArm, thresholds) { + const { minRecall, minPrecision } = thresholds; + const recall = judgeArm.recall ?? 0; + const precision = judgeArm.precision; // may be null + const staleRecall = stalenessArm.recall ?? 0; + + const recallOk = recall >= minRecall; + const precisionOk = precision != null && precision >= minPrecision; + const beatsStaleness = recall > staleRecall; + const pass = recallOk && precisionOk && beatsStaleness; + + const reasons = []; + if (!recallOk) reasons.push(`recall ${fmt(recall)} < minRecall ${minRecall}`); + if (!precisionOk) { + reasons.push( + precision == null + ? 'judge flagged nothing (precision undefined)' + : `precision ${fmt(precision)} < minPrecision ${minPrecision}`, + ); + } + if (!beatsStaleness) { + reasons.push(`recall ${fmt(recall)} does not beat staleness ${fmt(staleRecall)}`); + } + if (pass) reasons.push('all criteria met'); + return { pass, recallOk, precisionOk, beatsStaleness, thresholds, reasons }; +} + +function fmt(x) { + return x == null ? 'n/a' : x.toFixed(3); +} + +// source_silent is the judge's "I fetched but the page does not state this value" +// verdict. It is diagnostic, not a flag. Two populations matter: +// - on verifiable claims (the human DID find a value): every source_silent is a +// judge verification miss. Split by whether the gold value was correct vs an error +// (a source_silent on a real error is a missed catch via "can't verify"). +// - on unsourced-in-P claims (the human ALSO found no value): source_silent here is +// agreement — a good sign the judge reproduces the unverifiable boundary. +function sourceSilentDiagnostics(P) { + let onVerifiableNegative = 0; + let onVerifiableError = 0; + let agreesWithUnsourced = 0; + let disagreesWithUnsourced = 0; + for (const c of P) { + const silent = c.judge_verdict === 'source_silent'; + if (c.verdict === 'unsourced') { + if (silent) agreesWithUnsourced += 1; + else disagreesWithUnsourced += 1; + } else if (silent) { + if (ERROR_VERDICTS.has(c.verdict)) onVerifiableError += 1; + else onVerifiableNegative += 1; + } + } + return { + onVerifiableNegative, + onVerifiableError, + agreesWithUnsourced, + disagreesWithUnsourced, + }; +} + +// Per claim_type judge confusion matrix over the verifiable subset (so the report can +// show where the judge earns its keep — sku/version/tpm carry the highest error rates). +function judgeByClaimType(verifiable) { + const byType = {}; + for (const c of verifiable) { + (byType[c.claim_type] ||= []).push(c); + } + const out = {}; + for (const [t, arr] of Object.entries(byType)) { + out[t] = gradeArm(arr, judgeFlag); + } + return out; +} + +// Top-level: join already done by the caller (each claim carries judge_verdict). +// Filters to P, splits verifiable vs unsourced, grades all three arms over the same +// verifiable set, computes diagnostics + the gate, returns one report object. +export function computeBakeoff(joinedClaims, thresholds) { + const P = evalPopulation(joinedClaims); + const verifiable = P.filter((c) => c.verdict !== 'unsourced'); + const unsourcedInP = P.length - verifiable.length; + + const arms = { + staleness: gradeArm(verifiable, stalenessFlag), + judge: gradeArm(verifiable, judgeFlag), + hybrid: gradeArm(verifiable, hybridFlag), + }; + const gate = gateDecision(arms.judge, arms.staleness, thresholds); + + return { + population: { + total: P.length, + verifiable: verifiable.length, + positives: verifiable.filter((c) => ERROR_VERDICTS.has(c.verdict)).length, + negatives: verifiable.filter((c) => c.verdict === 'correct').length, + unsourcedInP, + }, + arms, + sourceSilent: sourceSilentDiagnostics(P), + byClaimType: judgeByClaimType(verifiable), + gate, + }; +} diff --git a/scripts/kb-eval/run-judge-bakeoff.mjs b/scripts/kb-eval/run-judge-bakeoff.mjs new file mode 100644 index 0000000..326a456 --- /dev/null +++ b/scripts/kb-eval/run-judge-bakeoff.mjs @@ -0,0 +1,155 @@ +#!/usr/bin/env node +// run-judge-bakeoff.mjs — S1 final glue: join the blind judge results back to the +// frozen gold set, grade the judge / staleness / hybrid arms, apply the +// PRE-REGISTERED gate, and write the bake-off report. All math lives in tested +// lib/judge-bakeoff.mjs; this CLI is thin wiring (same shape as compute-base-rate.mjs). +// +// The thresholds are required flags, not defaults: the gate must be locked BEFORE the +// judge fan-out runs (pre-registration), and the locked values are recorded in the +// report _meta so the decision is auditable. +// +// Usage: +// node scripts/kb-eval/run-judge-bakeoff.mjs --min-recall 0.70 --min-precision 0.60 [--write] +// +// Inputs: data/gold-correctness-set.json (answer key) +// data/judge-bakeoff-results.json ({ results: [{id, judge_verdict, ...}] }) +// Outputs: data/judge-bakeoff-report.{json,md} (with --write) + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { computeBakeoff, evalPopulation } from './lib/judge-bakeoff.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const DATA = path.join(__dirname, 'data'); + +function flag(name) { + const i = process.argv.indexOf(name); + return i >= 0 ? process.argv[i + 1] : undefined; +} +const minRecall = Number(flag('--min-recall')); +const minPrecision = Number(flag('--min-precision')); +if (!Number.isFinite(minRecall) || !Number.isFinite(minPrecision)) { + console.error('error: --min-recall and --min-precision are required (pre-registered gate)'); + process.exit(2); +} +const thresholds = { minRecall, minPrecision }; + +const gold = JSON.parse(fs.readFileSync(path.join(DATA, 'gold-correctness-set.json'), 'utf8')); +const resultsPath = path.join(DATA, 'judge-bakeoff-results.json'); +if (!fs.existsSync(resultsPath)) { + console.error(`error: ${resultsPath} not found — run the judge fan-out first`); + process.exit(2); +} +const judge = JSON.parse(fs.readFileSync(resultsPath, 'utf8')); + +// Join judge verdicts back to gold by id. +const verdictById = new Map((judge.results || []).map((r) => [r.id, r.judge_verdict])); +const joined = gold.claims.map((c) => ({ ...c, judge_verdict: verdictById.get(c.id) })); + +// Completeness check: every claim in P must have a judge verdict. +const missing = evalPopulation(joined).filter((c) => c.judge_verdict == null); +if (missing.length && !process.argv.includes('--allow-incomplete')) { + console.error(`error: ${missing.length} P-claims have no judge verdict (incomplete fan-out).`); + console.error('first few:', missing.slice(0, 5).map((c) => c.id).join(', ')); + console.error('pass --allow-incomplete to grade anyway (missing claims count as not-flagged).'); + process.exit(2); +} + +const r = computeBakeoff(joined, thresholds); + +const pct = (x) => (x == null ? 'n/a' : `${(x * 100).toFixed(1)}%`); +const num = (x) => (x == null ? 'n/a' : x.toFixed(3)); +const band = (w) => (w == null ? 'n/a' : `[${pct(w.low)}, ${pct(w.high)}]`); + +function armRow(name, a) { + return `| ${name} | ${a.tp} | ${a.fp} | ${a.fn} | ${a.tn} | ${pct(a.precision)} | ${pct(a.recall)} | ${band(a.recallWilson)} | ${num(a.f1)} |`; +} +function typeRow(t, a) { + return `| ${t} | ${a.positives} | ${a.tp} | ${a.fp} | ${a.fn} | ${pct(a.precision)} | ${pct(a.recall)} |`; +} + +const p = r.population; +const g = r.gate; +const ss = r.sourceSilent; +const verdict = g.pass ? '✅ PASS — bygg S3' : '❌ FAIL — stopp, ikke bygg S3'; + +const md = `# Judge bake-off-rapport — S1 (Fase 3 de-risk) + +_Generert deterministisk av \`run-judge-bakeoff.mjs\` over \`gold-correctness-set.json\` + \`judge-bakeoff-results.json\`. Tall fra testet \`lib/judge-bakeoff.mjs\`. Ikke rediger for hånd — regenerer._ + +**Forhåndsregistrert gate (låst FØR fan-out):** recall ≥ ${minRecall}, presisjon ≥ ${minPrecision}, OG judge-recall > staleness-recall. + +## Evaluerings-populasjon (P) + +Volatil stratum + fetchbare claim_types (price ekskludert) — der feilene bor; unngår «invertert leverage». + +| metrikk | verdi | +|---|---| +| P totalt | ${p.total} | +| Verifiserbare (correct/outdated/wrong) | ${p.verifiable} | +| Positive (reelle feil å fange) | ${p.positives} | +| Negative (correct) | ${p.negatives} | +| Unsourced i P (kjørt, men utenfor P/R) | ${p.unsourcedInP} | + +## Arm-sammenligning (detektering over de ${p.verifiable} verifiserbare) + +| arm | TP | FP | FN | TN | presisjon | recall | recall Wilson 95% | F1 | +|---|---|---|---|---|---|---|---|---| +${armRow('staleness (billig baseline)', r.arms.staleness)} +${armRow('judge (per-påstand groundedness)', r.arms.judge)} +${armRow('hybrid (union)', r.arms.hybrid)} + +## Judge per claim_type (verifiserbar delmengde) + +| claim_type | positive | TP | FP | FN | presisjon | recall | +|---|---|---|---|---|---|---| +${Object.entries(r.byClaimType).sort((a, b) => b[1].positives - a[1].positives).map(([t, a]) => typeRow(t, a)).join('\n')} + +## source_silent-diagnostikk + +Judgen hentet siden men fant ikke verdien. Diagnostisk, ikke et flagg. + +| signal | antall | tolkning | +|---|---|---| +| På verifiserbar feil | ${ss.onVerifiableError} | judge-bom: reell feil oversett via «kan ikke verifisere» | +| På verifiserbar correct | ${ss.onVerifiableNegative} | judge reproduserte ikke et korrekt faktum mennesket fant | +| Enig med unsourced | ${ss.agreesWithUnsourced} | judge reproduserer den uverifiserbare grensen (godt) | +| Uenig med unsourced | ${ss.disagreesWithUnsourced} | judge hevdet grunnet/ugrunnet der mennesket ikke fant kilde | + +## GATE: ${verdict} + +- recall ${num(r.arms.judge.recall)} ≥ ${minRecall}? **${g.recallOk ? 'ja' : 'nei'}** +- presisjon ${num(r.arms.judge.precision)} ≥ ${minPrecision}? **${g.precisionOk ? 'ja' : 'nei'}** +- slår staleness (recall ${num(r.arms.staleness.recall)})? **${g.beatsStaleness ? 'ja' : 'nei'}** +- begrunnelse: ${g.reasons.join('; ')} +`; + +if (process.argv.includes('--write')) { + const jsonOut = path.join(DATA, 'judge-bakeoff-report.json'); + const mdOut = path.join(DATA, 'judge-bakeoff-report.md'); + fs.writeFileSync( + jsonOut, + JSON.stringify( + { + _meta: { + source: 'gold-correctness-set.json + judge-bakeoff-results.json', + thresholds, + judged: (judge.results || []).length, + }, + ...r, + }, + null, + 2, + ) + '\n', + ); + fs.writeFileSync(mdOut, md); + console.log(`wrote ${jsonOut}`); + console.log(`wrote ${mdOut}`); +} else { + console.log(`P=${p.total} verifiable=${p.verifiable} positives=${p.positives}`); + console.log(`judge: precision=${num(r.arms.judge.precision)} recall=${num(r.arms.judge.recall)} f1=${num(r.arms.judge.f1)}`); + console.log(`staleness: recall=${num(r.arms.staleness.recall)}`); + console.log(`GATE: ${g.pass ? 'PASS' : 'FAIL'} — ${g.reasons.join('; ')}`); + console.log('(dry run — pass --write to persist judge-bakeoff-report.json + .md)'); +} diff --git a/tests/kb-eval/test-judge-bakeoff.test.mjs b/tests/kb-eval/test-judge-bakeoff.test.mjs new file mode 100644 index 0000000..b6c3120 --- /dev/null +++ b/tests/kb-eval/test-judge-bakeoff.test.mjs @@ -0,0 +1,199 @@ +// tests/kb-eval/test-judge-bakeoff.test.mjs +// Unit tests for the S1 judge bake-off aggregation lib. +// +// S1 de-risks the Fase 3 correctness judge BEFORE any production scaffolding is +// built: run a per-claim groundedness judge against the frozen 373-claim gold set +// and ask whether it beats the cheap staleness baseline (recall 0/40) by enough to +// justify ~2700 non-batchable fetches per full pass. +// +// The eval population P is volatile + fetchable claim_types (price excluded) — the +// judge is measured exactly where the real errors live, which structurally avoids +// the "inverted leverage" trap (a judge that wins only by auto-scoring cheap stable +// claims). Grading is a pure detection confusion matrix vs the gold verdicts: +// gold positive = verdict ∈ {outdated, wrong} (a real error to catch) +// gold negative = verdict === 'correct' +// excluded = verdict === 'unsourced' (no ground-truth value) +// predicted positive (flag) = judge_verdict === 'not_grounded' +// All computation is deterministic — no I/O, no Date.now/Math.random. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + evalPopulation, + blindClaims, + gradeArm, + stalenessFlag, + judgeFlag, + hybridFlag, + gateDecision, + computeBakeoff, +} from '../../scripts/kb-eval/lib/judge-bakeoff.mjs'; + +// --- synthetic joined gold+judge set ---------------------------------------- +// Each claim carries its gold fields plus the (blind) judge_verdict the harness +// joined back by id. Comments give the intended confusion-matrix cell. +const J = [ + // 1: volatile/version, correct, judge grounded -> P, verifiable, TN + { id: 'a#1', skill: 'x', stratum: 'volatile', claim_type: 'version', verdict: 'correct', lastmod_changed: false, judge_verdict: 'grounded' }, + // 2: volatile/sku, outdated, judge not_grounded -> P, TP (judge catches) + { id: 'a#2', skill: 'x', stratum: 'volatile', claim_type: 'sku', verdict: 'outdated', lastmod_changed: false, judge_verdict: 'not_grounded' }, + // 3: volatile/tpm, wrong, judge grounded -> P, FN (judge misses) + { id: 'a#3', skill: 'x', stratum: 'volatile', claim_type: 'tpm', verdict: 'wrong', lastmod_changed: false, judge_verdict: 'grounded' }, + // 4: volatile/status, correct, judge not_grounded -> P, FP (false alarm) + { id: 'a#4', skill: 'x', stratum: 'volatile', claim_type: 'status', verdict: 'correct', lastmod_changed: false, judge_verdict: 'not_grounded' }, + // 5: volatile/region, outdated, lastmod TRUE, judge ng -> P, TP; staleness also catches + { id: 'a#5', skill: 'x', stratum: 'volatile', claim_type: 'region', verdict: 'outdated', lastmod_changed: true, judge_verdict: 'not_grounded' }, + // 6: volatile/PRICE, wrong -> excluded from P (price) + { id: 'a#6', skill: 'x', stratum: 'volatile', claim_type: 'price', verdict: 'wrong', lastmod_changed: false, judge_verdict: 'not_grounded' }, + // 7: CONTROL/taxonomy, wrong -> excluded from P (control stratum) + { id: 'a#7', skill: 'x', stratum: 'control', claim_type: 'taxonomy', verdict: 'wrong', lastmod_changed: false, judge_verdict: 'not_grounded' }, + // 8: volatile/status, unsourced, judge source_silent -> P but not verifiable (excluded from P/R) + { id: 'a#8', skill: 'x', stratum: 'volatile', claim_type: 'status', verdict: 'unsourced', lastmod_changed: false, judge_verdict: 'source_silent' }, + // 9: volatile/taxonomy, correct, judge source_silent -> P, verifiable negative, not flagged -> TN + { id: 'a#9', skill: 'x', stratum: 'volatile', claim_type: 'taxonomy', verdict: 'correct', lastmod_changed: false, judge_verdict: 'source_silent' }, +]; + +// --- evalPopulation ---------------------------------------------------------- +test('evalPopulation — keeps volatile + fetchable, drops price and control', () => { + const P = evalPopulation(J); + const ids = P.map((c) => c.id).sort(); + assert.deepEqual(ids, ['a#1', 'a#2', 'a#3', 'a#4', 'a#5', 'a#8', 'a#9']); +}); + +// --- blindClaims (no label leakage) ----------------------------------------- +test('blindClaims — emits all of P with only the allowed blind fields', () => { + const blind = blindClaims(J); + assert.equal(blind.length, 7); // all of P, unsourced included + const allowed = new Set(['id', 'file', 'skill', 'claim', 'claim_type', 'evidence_url']); + for (const c of blind) { + for (const k of Object.keys(c)) { + assert.ok(allowed.has(k), `leaked field: ${k}`); + } + // the answer key must never appear + assert.equal(c.verdict, undefined); + assert.equal(c.notes, undefined); + assert.equal(c.lastmod_changed, undefined); + assert.equal(c.stratum, undefined); + } +}); + +// --- flag predicates --------------------------------------------------------- +test('stalenessFlag — true only when lastmod_changed === true', () => { + assert.equal(stalenessFlag({ lastmod_changed: true }), true); + assert.equal(stalenessFlag({ lastmod_changed: false }), false); + assert.equal(stalenessFlag({ lastmod_changed: null }), false); +}); +test('judgeFlag — true only when judge_verdict === not_grounded', () => { + assert.equal(judgeFlag({ judge_verdict: 'not_grounded' }), true); + assert.equal(judgeFlag({ judge_verdict: 'grounded' }), false); + assert.equal(judgeFlag({ judge_verdict: 'source_silent' }), false); +}); +test('hybridFlag — union of staleness and judge', () => { + assert.equal(hybridFlag({ lastmod_changed: true, judge_verdict: 'grounded' }), true); + assert.equal(hybridFlag({ lastmod_changed: false, judge_verdict: 'not_grounded' }), true); + assert.equal(hybridFlag({ lastmod_changed: false, judge_verdict: 'source_silent' }), false); +}); + +// --- gradeArm ---------------------------------------------------------------- +// Verifiable subset of P = ids 1,2,3,4,5,9 (8 excluded: unsourced). +function verifiableP() { + return evalPopulation(J).filter((c) => c.verdict !== 'unsourced'); +} + +test('gradeArm — judge arm confusion matrix', () => { + const a = gradeArm(verifiableP(), judgeFlag); + // positives = outdated/wrong = {2,3,5}=3 ; negatives = correct = {1,4,9}=3 + assert.equal(a.positives, 3); + assert.equal(a.negatives, 3); + // flags: 2(TP),4(FP),5(TP) ; misses: 3(FN); correct-rejects: 1,9 (TN) + assert.equal(a.tp, 2); + assert.equal(a.fp, 1); + assert.equal(a.fn, 1); + assert.equal(a.tn, 2); + assert.ok(Math.abs(a.precision - 2 / 3) < 1e-9); + assert.ok(Math.abs(a.recall - 2 / 3) < 1e-9); + assert.ok(Math.abs(a.f1 - 2 / 3) < 1e-9); +}); + +test('gradeArm — staleness arm catches only the lastmod-changed error', () => { + const a = gradeArm(verifiableP(), stalenessFlag); + // only id 5 has lastmod true (a positive) -> tp=1; positives 2,3 missed -> fn=2 + assert.equal(a.tp, 1); + assert.equal(a.fp, 0); + assert.equal(a.fn, 2); + assert.equal(a.tn, 3); + assert.ok(Math.abs(a.recall - 1 / 3) < 1e-9); + assert.equal(a.precision, 1); // 1 flag, 1 right +}); + +test('gradeArm — precision is null when nothing is flagged', () => { + const noFlag = gradeArm(verifiableP(), () => false); + assert.equal(noFlag.tp, 0); + assert.equal(noFlag.fp, 0); + assert.equal(noFlag.precision, null); // 0/0 -> null, not NaN + assert.equal(noFlag.recall, 0); + assert.equal(noFlag.f1, null); + assert.equal(noFlag.precisionWilson, null); // no flags -> no band +}); + +test('gradeArm — Wilson bands bracket the point estimates', () => { + const a = gradeArm(verifiableP(), judgeFlag); // recall .667 (2/3), precision .667 (2/3) + assert.ok(a.recallWilson.low <= a.recall && a.recall <= a.recallWilson.high); + assert.ok(a.precisionWilson.low <= a.precision && a.precision <= a.precisionWilson.high); +}); + +// --- gateDecision ------------------------------------------------------------ +test('gateDecision — passes when judge clears both thresholds and beats staleness', () => { + const judge = gradeArm(verifiableP(), judgeFlag); // recall .667 prec .667 + const stale = gradeArm(verifiableP(), stalenessFlag); // recall .333 + const g = gateDecision(judge, stale, { minRecall: 0.6, minPrecision: 0.6 }); + assert.equal(g.pass, true); + assert.equal(g.beatsStaleness, true); + assert.equal(g.recallOk, true); + assert.equal(g.precisionOk, true); +}); + +test('gateDecision — fails when recall below threshold', () => { + const judge = gradeArm(verifiableP(), judgeFlag); // recall .667 + const stale = gradeArm(verifiableP(), stalenessFlag); + const g = gateDecision(judge, stale, { minRecall: 0.7, minPrecision: 0.6 }); + assert.equal(g.pass, false); + assert.equal(g.recallOk, false); + assert.equal(g.precisionOk, true); + assert.ok(Array.isArray(g.reasons) && g.reasons.length >= 1); +}); + +test('gateDecision — beatsStaleness requires strictly greater recall', () => { + const judge = gradeArm(verifiableP(), judgeFlag); // recall .667 + // staleness with identical recall -> not strictly greater + const fakeStale = { recall: 0.667, precision: 1, tp: 2, fp: 0, fn: 1, tn: 2, positives: 3, negatives: 3 }; + const g = gateDecision(judge, fakeStale, { minRecall: 0.6, minPrecision: 0.6 }); + assert.equal(g.beatsStaleness, false); + assert.equal(g.pass, false); +}); + +// --- computeBakeoff (top-level) --------------------------------------------- +test('computeBakeoff — wires population, three arms, diagnostics and gate', () => { + const r = computeBakeoff(J, { minRecall: 0.6, minPrecision: 0.6 }); + assert.equal(r.population.total, 7); // P + assert.equal(r.population.verifiable, 6); + assert.equal(r.population.positives, 3); + assert.equal(r.population.unsourcedInP, 1); // id 8 + // three arms present + assert.ok(r.arms.judge && r.arms.staleness && r.arms.hybrid); + assert.equal(r.arms.judge.tp, 2); + // hybrid = union; here same recall as judge (staleness adds the already-caught #5) + assert.equal(r.arms.hybrid.tp, 2); + // source_silent diagnostics + assert.equal(r.sourceSilent.onVerifiableNegative, 1); // id 9 correct but judge couldn't verify + assert.equal(r.sourceSilent.agreesWithUnsourced, 1); // id 8 unsourced, judge source_silent + // gate + assert.equal(r.gate.pass, true); +}); + +test('computeBakeoff — per claim_type judge breakdown present', () => { + const r = computeBakeoff(J, { minRecall: 0.6, minPrecision: 0.6 }); + // version/sku/tpm/status/region/taxonomy each appear in verifiable P + assert.ok(r.byClaimType.sku); + assert.equal(r.byClaimType.sku.tp, 1); // id 2 outdated sku, judge ng +});