From 706f44fb34f48cfe038b3e43e8af98c8396dcaa6 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Sat, 18 Jul 2026 17:34:49 +0200 Subject: [PATCH] =?UTF-8?q?feat(ms-ai-architect):=20R7.1=20=E2=80=94=20jud?= =?UTF-8?q?ge-pass=2030/45=20filer=20(362/544=20claims;=201=20pass,=2029?= =?UTF-8?q?=20flagged,=20106=20flagg;=20v3.1-judge,=20ledger=20durabel=20p?= =?UTF-8?q?er=20fil)=20[skip-docs]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/kb-eval/data/judge-pass-manifest.json | 1613 +++++++++++++++++ scripts/kb-eval/data/r71-payloads.json | 227 +++ 2 files changed, 1840 insertions(+) create mode 100644 scripts/kb-eval/data/judge-pass-manifest.json create mode 100644 scripts/kb-eval/data/r71-payloads.json diff --git a/scripts/kb-eval/data/judge-pass-manifest.json b/scripts/kb-eval/data/judge-pass-manifest.json new file mode 100644 index 0000000..fbe84e9 --- /dev/null +++ b/scripts/kb-eval/data/judge-pass-manifest.json @@ -0,0 +1,1613 @@ +{ + "_meta": { + "purpose": "R7–R10 full-pass judge ledger: one record per never-verified reference file — born-verified pass (surgical stamp) or flagged (R11 work-list). Durable per file.", + "derived_from": "scripts/kb-update/data/full-pass-worklist.json (243 due, all never-verified)", + "cadence": "R7–R10 (5 økter × ~49, 8–10 samtidige)", + "batch": "R7.1", + "count": 30, + "generated": "2026-07-18" + }, + "files": [ + { + "file": "skills/ms-ai-engineering/references/agent-orchestration/foundry-workflows-visual-orchestration.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 16, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#2", + "judge_verdict": "not_grounded", + "rule": "R3", + "evidence_url": "https://learn.microsoft.com/azure/foundry/agents/concepts/workflow", + "evidence_quote": "Because workflows are no longer a separate agent type, you run the result by deploying it as a hosted agent", + "reason": "The claim's three-agent-type taxonomy (Prompt-based | Workflow | Hosted) is superseded: the live docs state workflows are no longer a separate agent type and the development-lifecycle page enumerates exactly two agent types (Prompt-based and Hosted).", + "file": "skills/ms-ai-engineering/references/agent-orchestration/foundry-workflows-visual-orchestration.md", + "line": 33, + "claim": "Workflows er ett av tre agenttyper i Foundry: Prompt-based | Workflow | Hosted (preview, containeriserte agenter)", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#6", + "judge_verdict": "not_grounded", + "rule": "R2", + "evidence_url": "https://learn.microsoft.com/azure/foundry/agents/concepts/tool-catalog", + "evidence_quote": "The following table lists all built-in tools available in Foundry Agent Service. | Web search | Code Interpreter | Custom Code Interpreter (preview) | File Search | Azure AI Search | Azure Functions | Function calling | Image Generation (preview) | Browser Automation (preview) | Computer Use (preview) | Microsoft Fabric (preview) | SharePoint (preview)", + "reason": "The canonical tool catalog enumerates the built-in and custom tools (Code Interpreter, Azure AI Search, MCP, and Bing/Web search are present) but Key Vault is absent — in Foundry docs Key Vault is a secret-store connection, not a tool in the agent tool catalog, so one asserted list member does not exist.", + "file": "skills/ms-ai-engineering/references/agent-orchestration/foundry-workflows-visual-orchestration.md", + "line": 274, + "claim": "Verktøy tilgjengelige i agent-noder i workflows (samme katalog som enkelt-agenter): Code Interpreter, Bing Search, Azure AI Search, Key Vault, MCP-servere", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#8", + "judge_verdict": "not_grounded", + "rule": "R3", + "evidence_url": "https://learn.microsoft.com/azure/logic-apps/add-agent-action-create-run-workflow", + "evidence_quote": "Azure Logic Apps supports 1,400+ connectors and native, built-in data operations, so agents can integrate with many Microsoft and non-Microsoft services or products.", + "reason": "The docs attach the 1,400+ figure to connectors, not triggers, and state no '400+ enterprise connectors' count anywhere — the enterprise-connector category is described as a small set (SAP, IBM MQ, IBM 3270) with Salesforce classified as a Standard connector, so the claim's organizing units and figures do not match the source.", + "file": "skills/ms-ai-engineering/references/agent-orchestration/foundry-workflows-visual-orchestration.md", + "line": 306, + "claim": "Azure Logic Apps: 1400+ triggere (HTTP, Events, Schedule, Queues, SaaS) | 400+ enterprise-koblinger (SAP, Salesforce, AS2, EDI)", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#14", + "judge_verdict": "not_grounded", + "rule": "R2", + "evidence_url": "https://learn.microsoft.com/azure/foundry/observability/how-to/how-to-monitor-agents-dashboard?view=foundry", + "evidence_quote": "Use these definitions to interpret the dashboard: Token usage ... Latency ... Run success rate ... Evaluation metrics ... Red teaming results", + "reason": "The dashboard's canonical metric enumeration lists Token usage, Latency, Run success rate, Evaluation metrics, and Red teaming results — 'Error rate' is not among the dashboard metrics (it appears only in a note about onboarding external custom agents), so one asserted list member is absent.", + "file": "skills/ms-ai-engineering/references/agent-orchestration/foundry-workflows-visual-orchestration.md", + "line": 391, + "claim": "Agent Monitoring Dashboard-metrikker: Token usage | Latency | Run success rate | Error rate | Evaluation metrics", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/agent-orchestration/semantic-kernel-agents-implementation.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 8, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/agent-orchestration/semantic-kernel-agents-implementation.md#1", + "judge_verdict": "not_grounded", + "rule": "R2", + "evidence_url": "https://learn.microsoft.com/agent-framework/overview/agent-framework-overview", + "evidence_quote": "dotnet add package Microsoft.Agents.AI.Foundry --prerelease", + "reason": "The canonical overview and support pages state no '1.0' version, no GA status, and no 2026-04-03 date anywhere on Learn, and the live getting-started still instructs installing prerelease packages (--prerelease for .NET, 'pip install --pre agent-framework-foundry' in the migration guide; Go explicitly 'in public preview'), so the asserted GA/production-ready status is absent from the pages that would state it and contradicted by current prerelease packaging.", + "file": "skills/ms-ai-engineering/references/agent-orchestration/semantic-kernel-agents-implementation.md", + "line": 4, + "claim": "Microsoft Agent Framework 1.0 nådde GA 3. april 2026; produksjonsklart open-source rammeverk (.NET + Python)", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/agent-orchestration/semantic-kernel-agents-implementation.md#3", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/agent-framework/workflows/", + "evidence_quote": "Multi-Agent Orchestration: Built-in patterns for coordinating multiple AI agents, including sequential, concurrent, hand-off, and magentic.", + "reason": "The MAF workflows page lists only four built-in patterns (Group Chat is absent) and states no 'stable (GA) in MAF 1.0' status anywhere, while the cited Semantic Kernel orchestration page carries the banner 'Agent Orchestration features in the Agent Framework are in the experimental stage' — the five-pattern and GA-stability load-bearing parts both fail even though checkpointing/HITL are documented.", + "file": "skills/ms-ai-engineering/references/agent-orchestration/semantic-kernel-agents-implementation.md", + "line": 9, + "claim": "De fem orkestreringsmønstrene Sequential, Concurrent, Handoff, Group Chat og Magentic er stabile (GA) i MAF 1.0 med streaming, checkpointing, human-in-the-loop og pause/resume", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/agent-orchestration/semantic-kernel-agents-implementation.md#8", + "judge_verdict": "source_silent", + "rule": "", + "evidence_url": "https://learn.microsoft.com/agent-framework/support/", + "evidence_quote": "", + "reason": "Learn pages confirm both are open-source with GitHub repositories but no learn.microsoft.com page states the MIT license for either Semantic Kernel or Agent Framework; the license terms live in the GitHub repos, which are outside the allowed source.", + "file": "skills/ms-ai-engineering/references/agent-orchestration/semantic-kernel-agents-implementation.md", + "line": 417, + "claim": "Semantic Kernel: MIT License | Microsoft Agent Framework: MIT License", + "disposition": "unsourced" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/api-management/apim-ai-gateway-overview.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 7, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/api-management/apim-ai-gateway-overview.md#2", + "judge_verdict": "not_grounded", + "rule": "R3", + "evidence_url": "https://learn.microsoft.com/azure/api-management/api-management-policies", + "evidence_quote": "Limit large language model API token usage | Prevents large language model (LLM) API usage spikes by limiting LLM tokens per calculated key.", + "reason": "The canonical policy reference's AI gateway and rate-limiting tables enumerate only llm-token-limit, llm-emit-token-metric, llm-semantic-cache-lookup/store and llm-content-safety — azure-openai-token-limit and azure-openai-emit-token-metric are absent, and their doc URLs now redirect to the llm-* pages (which cover OpenAI, Anthropic and Google Vertex APIs), so the claimed dual azure-openai-*/llm-* taxonomy describes a superseded policy split.", + "file": "skills/ms-ai-engineering/references/api-management/apim-ai-gateway-overview.md", + "line": 82, + "claim": "Token-policies i APIM: azure-openai-token-limit (Azure OpenAI) | llm-token-limit (alle LLM-er) | azure-openai-emit-token-metric | llm-emit-token-metric", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/api-management/apim-authentication-oauth-managed-identity.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 6, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/api-management/apim-authentication-oauth-managed-identity.md#2", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/role-based-access-control", + "evidence_quote": "Cognitive Services OpenAI User - Cognitive Services OpenAI Contributor - Cognitive Services Contributor - Cognitive Services Usages Reader", + "reason": "The canonical Azure OpenAI RBAC page enumerates the fourth role as 'Cognitive Services Usages Reader' (subscription-level quota viewing), not a generic 'Reader (lese)' role, and shows Cognitive Services Contributor is not 'full tilgang' (it cannot access quota or make inference API calls with Microsoft Entra ID) — two stated load-bearing parts fail, so the whole claim is not grounded.", + "file": "skills/ms-ai-engineering/references/api-management/apim-authentication-oauth-managed-identity.md", + "line": 94, + "claim": "RBAC-roller for Azure OpenAI: Cognitive Services OpenAI User (bruke deployments) | Cognitive Services OpenAI Contributor (opprette/administrere deployments) | Cognitive Services Contributor (full tilgang) | Reader (lese)", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/api-management/backend-pool-management.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 10, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/api-management/backend-pool-management.md#10", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/azure-monitor/reference/tables/apimanagementgatewayllmlog", + "evidence_quote": "ApiVersion, CompletionTokens, CorrelationId, DeploymentName, IsStreamCompletion, ModelName, OperationName, PromptTokens, Region, RequestId, RequestMessages, ResponseMessages, SequenceNumber, TimeGenerated, TotalTokens, Type", + "reason": "The canonical ApiManagementGatewayLlmLog schema page enumerates all columns and confirms TotalTokens, PromptTokens, and CompletionTokens, but no BackendUrl column exists in that enumeration — a stated, load-bearing column name a reader would copy into a KQL query is wrong, so the whole claim fails under R8/R2 despite the other parts holding.", + "file": "skills/ms-ai-engineering/references/api-management/backend-pool-management.md", + "line": 405, + "claim": "Log Analytics-tabellen ApiManagementGatewayLlmLog finnes med kolonner TotalTokens, PromptTokens, CompletionTokens, BackendUrl (og ApiManagementGatewayLogs med BackendResponseCode)", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-api-best-practices.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 12, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#1", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/supported-languages#error-handling", + "evidence_quote": "The following errors are automatically retired twice by default with a brief exponential backoff: Connection Errors, 408 Request Timeout, 429 Rate Limit, >=500 Internal Errors", + "reason": "The claim asserts Python, .NET and Go all retry up to 3 times, but the live page states the Python (and JS) OpenAI SDK retries twice by default — only the .NET client classes retry 'up to three more times' — so a load-bearing part is contradicted.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-api-best-practices.md", + "line": 87, + "claim": "Azure OpenAI SDK-er (Python, .NET, Go) retrier automatisk 408|429|500|502|503|504 — opptil 3 ganger med eksponentiell backoff", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#10", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/provisioned-throughput-billing", + "evidence_quote": "With Reservations, you commit to payment for a fixed number of PTUs over a one-month or one-year term, and in return, you receive a discounted effective $/PTU/hr rate.", + "reason": "The claim asserts 1-year (~37%) and 3-year (~57%) commitment discounts, but the live billing page states reservation terms are one-month or one-year only — no 3-year term exists, and no discount percentages are documented ('The discount varies by model family and term length').", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-api-best-practices.md", + "line": 627, + "claim": "Provisioned reservation-rabatter: 1-årig commitment ~37%|3-årig commitment ~57%", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#11", + "judge_verdict": "not_grounded", + "rule": "R3", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/foundry-models/supported-languages", + "evidence_quote": "Azure AI Inference beta SDK is deprecated and will be retired on August 26, 2026. Switch to the generally available OpenAI/v1 API with a stable OpenAI SDK.", + "reason": "The claim's organizing frame — azure-ai-inference/Azure.AI.Inference/@azure/ai-inference as the Foundry-supporting SDKs — is superseded: the Azure AI Inference SDK family is deprecated (the migration table maps openai as the replacement for azure-ai-inference, retiring May 30, 2026) and Microsoft directs to the OpenAI SDKs, even though the Go openai-go part is still correct.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-api-best-practices.md", + "line": 482, + "claim": "SDK-er som støtter Microsoft Foundry: Python azure-ai-inference + openai (Azure-variant)|.NET Azure.AI.Inference + Azure.AI.OpenAI|JS/TS @azure/openai + @azure/ai-inference|Go github.com/openai/openai-go (med Azure endpoint)", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-cost-optimization.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 10, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#10", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/provisioned-throughput-sizing", + "evidence_quote": "The following Fireworks on Microsoft Foundry models support both Global and US Data Zone provisioned throughput.", + "reason": "The claim asserts PTU is only for Azure OpenAI, but the live sizing page publishes provisioned-throughput deployment parameters for non-OpenAI Foundry Models (Llama-3.3-70B-Instruct, DeepSeek-R1, DeepSeek-V3-0324, and the Fireworks model family), so the load-bearing 'kun for Azure OpenAI' exclusivity is superseded even though PTU still doesn't apply to classic AI Services like Speech or Vision.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-cost-optimization.md", + "line": 177, + "claim": "PTU er kun for Azure OpenAI — ikke for øvrige Azure AI Services", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 16, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#8", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/reliability/reliability-ai-search", + "evidence_quote": "Tier: Your service must be on the Basic tier or higher ... Number of replicas: Your service must have at least two replicas", + "reason": "The claim asserts Standard tier or higher plus minimum 3 replicas, but the live page states zone redundancy requires only Basic tier or higher and at least two replicas (3 replicas is the read-write SLA condition, not the zone-redundancy requirement), so two load-bearing values are contradicted even though the no-built-in-DR part holds.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "line": 81, + "claim": "Azure AI Search zone redundancy krever Standard tier eller høyere + minimum 3 replicas; ingen built-in disaster recovery (krever manuell gjenoppbygging eller support-kontakt)", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#11", + "judge_verdict": "not_grounded", + "rule": "", + "evidence_url": "https://learn.microsoft.com/azure/ai-foundry/openai/concepts/models#model-summary-table-and-region-availability", + "evidence_quote": "| norwayeast | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |", + "reason": "The claim says no Azure OpenAI region exists in Norway, but the canonical model/region availability table lists norwayeast with standard regional deployments of gpt-4.1, o3-mini, o1, gpt-4o and more (gpt-4o was available there well before 2026-02), directly refuting the non-existence claim.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "line": 364, + "claim": "Ingen Azure OpenAI-region i Norge per 2026-02; nærmeste regioner er Sweden Central og West Europe", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#12", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/api-management/api-management-features", + "evidence_quote": "Multi-region deployment | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✔️ | ❌ ... Scale units | ❌ (automatic scaling) | 1 | 2 | 10 | 4 | 10 | 12 per region | 30", + "reason": "The live tier comparison contradicts multiple load-bearing parts: multi-region deployment is Premium-only (the claim gives it to Standard), Basic allows 2 scale units (claim says 1 unit max), and Premium classic is 12 units per region; the per-tier call volumes are not stated on the page either.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "line": 435, + "claim": "APIM-tiers: Developer (1M calls, ingen SLA) | Basic (1M calls, SLA, 1 unit max) | Standard (10M calls, multi-region, 4 units) | Premium (unlimited, multi-region, VNet, 10+ units)", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#13", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities", + "evidence_quote": "APPLIES TO: All API Management tiers ... Currently, the backend circuit breaker isn't supported in the Consumption tier of API Management.", + "reason": "The circuit-breaker-not-in-Consumption part is confirmed on the backends page, but the second load-bearing part 'Standard tier minimum for AI gateway' is contradicted: the AI gateway page applies to all API Management tiers (with per-capability variation), so no Standard-minimum exists.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "line": 440, + "claim": "APIM circuit breaker er ikke tilgjengelig i Consumption tier; Standard tier minimum for AI gateway", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#14", + "judge_verdict": "not_grounded", + "rule": "", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/prompt-caching", + "evidence_quote": "Prompt caching is enabled by default for all supported models. There's no opt-out support for prompt caching.", + "reason": "The claim describes prompt caching for Azure OpenAI as a planned, not-yet-available feature, but the live page documents it as shipped, enabled by default, with discounted cache-read pricing and extended 24h retention on newer models — the status is superseded.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "line": 427, + "claim": "Caching (prompt caching) for Azure OpenAI omtales som planlagt feature (ikke tilgjengelig ennå)", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-governance-compliance.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 12, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#6", + "judge_verdict": "source_silent", + "rule": "", + "evidence_url": "https://learn.microsoft.com/en-us/azure/search/search-security-built-in", + "evidence_quote": "", + "reason": "The current data-residency section confirms object names (indexes, fields, indexers) appear in telemetry logs processed outside the selected region for Microsoft support, but no live learn.microsoft.com page states the claimed 1.5-year global retention period, so the specific retention value can be neither confirmed nor refuted.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-governance-compliance.md", + "line": 114, + "claim": "Telemetry logs (objektnavn som indexer, skillsets) lagres globalt i 1,5 år for Microsoft-support (unntak fra Geography-residency)", + "disposition": "unsourced" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#7", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/azure-monitor/logs/data-retention-configure", + "evidence_quote": "The default retention period of Analytics tables in a Log Analytics workspace is 30 days. You can change the default analytics period of Analytics tables up to two years by modifying the workspace-level data retention setting.", + "reason": "The claim asserts a 90-day Log Analytics default, but the live page states the default retention is 30 days (adjustable up to 730), contradicting the load-bearing default-value part even though the 30-730 range roughly holds.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-governance-compliance.md", + "line": 139, + "claim": "Diagnostic logs standard retention: 90 dager (Log Analytics default), justerbar 30-730 dager", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#9", + "judge_verdict": "not_grounded", + "rule": "", + "evidence_url": "https://learn.microsoft.com/azure/key-vault/managed-hsm/firmware-update", + "evidence_quote": "the HSM firmware for both Managed HSM and Azure Key Vault Premium are officially upgraded to a modern version validated to FIPS 140-3 level 3 standard", + "reason": "The claim asserts FIPS 140-2 Level 3 for HSM-backed Key Vault, but live docs state both Key Vault Premium and Managed HSM are now validated to FIPS 140-3 Level 3 (portfolio table lists both as FIPS 140-3 Level 3), so the certification version is superseded.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-governance-compliance.md", + "line": 220, + "claim": "Keys kan lagres i HSM-backed Key Vault for FIPS 140-2 Level 3", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#12", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/purview/dlp-microsoft-teams", + "evidence_quote": "Office 365 and Microsoft 365 E3 include DLP protection for SharePoint, OneDrive, and Exchange.", + "reason": "The claim's load-bearing part 'DLP Policies krever E5/F5-lisens' is contradicted — E3 includes DLP for SharePoint/OneDrive/Exchange (E5/F5 is only needed for Teams chat and Endpoint DLP), and the M365 license comparison also shows DSPM for AI included at E3/E5, undercutting the 'ikke i standard lisens' part, so the bundled licensing claim fails even though the audit-included and pay-as-you-go parts hold.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-governance-compliance.md", + "line": 557, + "claim": "Purview-lisensiering: DLP Policies krever E5/F5-lisens (eller pay-as-you-go) | Sensitivity Labels krever E3/E5 | DSPM for AI ikke i standard lisens (pay-as-you-go tilgjengelig) | Audit (Unified Log) inkludert", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-monitoring-logging.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 10, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-monitoring-logging.md#3", + "judge_verdict": "not_grounded", + "rule": "", + "evidence_url": "https://learn.microsoft.com/en-us/azure/azure-monitor/logs/data-retention-configure", + "evidence_quote": "By default, all tables in a Log Analytics workspace retain data for 30 days, except for log tables with 90-day default retention.", + "reason": "The claim says Log Analytics retention is 90 days free then paid, but the page states the default is 30 days (31 days included in the ingestion price), with 90-day free retention applying only to specific tables (Usage, AzureActivity, Application Insights App* tables).", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-monitoring-logging.md", + "line": 100, + "claim": "Log Analytics data retention: 90 dager gratis, deretter betalt", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-monitoring-logging.md#7", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/foundry-classic/openai/monitor-openai-reference", + "evidence_quote": "Azure OpenAI Requests: Number of calls made to the Azure OpenAI API over a period of time ... `AzureOpenAIRequests`", + "reason": "Four of the six names (TokenTransaction, GeneratedTokens, ProcessedPromptTokens, ActiveTokens) match the canonical metric reference, but the request metric is named `AzureOpenAIRequests` (not `Requests`) and no `Http429` metric exists anywhere in the enumeration (429s surface via the StatusCode dimension), so two load-bearing copy-into-code names are wrong.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-monitoring-logging.md", + "line": 294, + "claim": "Azure OpenAI metric-navn: TokenTransaction | GeneratedTokens | ProcessedPromptTokens | ActiveTokens (PTU) | Requests | Http429", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-monitoring-logging.md#8", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/azure-monitor/app/application-insights-faq", + "evidence_quote": "The default Pay-as-you-go Log Analytics pricing tier includes 5 GB per month of free data allowance per billing account.", + "reason": "The 5 GB/month free allowance is confirmed, but it is granted per billing account, not per subscription as the claim asserts - the scoping unit is a load-bearing part and it is contradicted.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-monitoring-logging.md", + "line": 394, + "claim": "Application Insights: 5 GB/måned gratis data ingestion per subscription", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-monitoring-logging.md#10", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/microsoft-copilot-studio/advanced-bot-framework-composer-capture-telemetry", + "evidence_quote": "Telemetry offers insights into your agent by tracking: Logged messages and events sent to and from your agent, Topics to trigger during user conversations, Custom telemetry events that you can send from your topics", + "reason": "The canonical Copilot Studio telemetry page confirms the App Insights connection and conversation analytics but enumerates only messages/events, topics, and custom events - no LUIS intent recognition or QnA Maker query latency (those belong to Bot Framework SDK bots, and QnA Maker was retired 31 March 2025), so two load-bearing parts fail.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-monitoring-logging.md", + "line": 332, + "claim": "Copilot Studio-bottar kan kobles til Application Insights for conversation analytics, LUIS intent recognition performance og QnA Maker query latency", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-networking-security.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 15, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#8", + "judge_verdict": "source_silent", + "rule": "", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/cognitive-services-virtual-networks", + "evidence_quote": "", + "reason": "Neither the cited networking page nor any Microsoft Learn page found via search states a pricing-tier prerequisite (S0/Basic or higher, Free tier excluded) for private endpoints/IP firewall/service endpoints on AI Services, so the tier requirement can be neither confirmed nor refuted.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-networking-security.md", + "line": 404, + "claim": "Nettverkssikkerhet (private endpoints, IP firewall, service endpoints) krever Basic tier (S0) eller høyere; Free tier ikke støttet", + "disposition": "unsourced" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#15", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/power-platform/admin/virtual-network-support-whitepaper", + "evidence_quote": "The new architecture eliminated costs and limitations associated with on-premises data gateways. ... a connector workload can access the target resource or endpoint inside the same virtual network.", + "reason": "The claim asserts Power Automate/Power Apps 'require' an on-premises data gateway and that custom connectors 'must' use the public endpoint, but live docs document Power Platform Virtual Network support (subnet delegation) letting connectors reach private-endpoint resources directly without a gateway — the 'krever'/'må' load-bearing parts describe a superseded state.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-networking-security.md", + "line": 316, + "claim": "Power Automate/Power Apps mot private endpoints krever On-premises data gateway; Azure Relay hybrid connection ikke støttet direkte; custom connector må bruke public endpoint med IP firewall", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-vs-foundry-tools-selection.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 16, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#2", + "judge_verdict": "not_grounded", + "rule": "", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/personalizer/what-is-personalizer", + "evidence_quote": "Starting 20 September 2023 you won't be able to create new Personalizer resources. The Personalizer service is being retired 1 October 2026.", + "reason": "The claim states Personalizer IS retired, but the live page states it is BEING retired on 1 October 2026 (still ~2.5 months away), i.e. deprecated/retiring, a different status value.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-vs-foundry-tools-selection.md", + "line": 58, + "claim": "Personalizer er utgått (retired)", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#6", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "evidence_quote": "| GPT-5.6 series | **NEW** `gpt-5.6-sol`, `gpt-5.6-terra`, `gpt-5.6-luna` |", + "reason": "The claim's load-bearing part 'GPT-5.5 nyest' is superseded: the live model page lists the GPT-5.6 series (2026-07-09) as the newest flagship series (and also a GPT-5.3 series the claim omits).", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-vs-foundry-tools-selection.md", + "line": 145, + "claim": "Azure OpenAI modellserie 2026-06: GPT-5-generasjonen (GPT-5/5.1/5.2/5.4/5.5 + -mini/-nano/-codex) er flaggskip for reasoning/chat; GPT-5.5 nyest", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#12", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/bot-service/file-format/bot-builder-qna-file-format", + "evidence_quote": "Azure AI QnA Maker will be retired on 31 March 2025. ... Custom question answering, a feature of Azure AI Language, is the updated version of the QnA Maker service.", + "reason": "The retirement part is correct (31 March 2025 has passed), but the load-bearing replacement part is wrong: the documented successor is custom question answering in Azure AI Language, not 'Language Understanding' (LUIS), which is itself a retired product.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-vs-foundry-tools-selection.md", + "line": 387, + "claim": "QnA Maker er utgått (retired); erstattes av Language Understanding for FAQ-bots", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#16", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/foundry/agents/how-to/agent-365", + "evidence_quote": "Microsoft 365 licensing that supports Microsoft Agent 365. Agent 365 works best with Microsoft E5, and at least one user in your organization must have a qualifying Microsoft Agent 365 license such as Microsoft 365 Copilot.", + "reason": "The Azure-subscription part holds, but the load-bearing licensing threshold is stated differently: the docs require a qualifying Agent 365 license such as Microsoft 365 Copilot (works best with E5), not 'M365 E3 or higher' — an E3 license alone does not meet the documented requirement.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-vs-foundry-tools-selection.md", + "line": 559, + "claim": "Agent-publisering til Agent 365 krever M365 E3 eller høyere (Azure subscription + M365 E3/E5)", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-image-analysis.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 14, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#9", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/faq", + "evidence_quote": "The free (S0) tier only allows 20 transactions per minute. Upgrade to the S1 tier to get up to 20 transactions per second.", + "reason": "The 'up to 20 requests/second on the Standard (S1) tier' part is grounded, but the stated '10-20 requests/sekund per tier' range is contradicted for the free tier, which allows only 20 transactions per minute (about 0.33/s), and no Learn page states a 10 rps tier for Azure AI Vision.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-image-analysis.md", + "line": 122, + "claim": "Rate limits: 10-20 requests/sekund per tier (opp til 20 requests/sekund per Standard tier)", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#10", + "judge_verdict": "not_grounded", + "rule": "", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/custom-vision-service/limits-and-quotas", + "evidence_quote": "Min labeled images per Tag, Classification (50+ recommended) | 5 | 5", + "reason": "The claim overstates a recommendation as a requirement: the Custom Vision limits page sets the enforced minimum at 5 labeled images per tag for classification and 15 for object detection (50+ recommended), while the 30-images figure appears in quickstarts only as guidance ('you should use at least 30 images per tag'), so Custom Vision does not require 30.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-image-analysis.md", + "line": 154, + "claim": "Custom Vision krever trening med minimum 30 bilder per tag", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#13", + "judge_verdict": "not_grounded", + "rule": "R2", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/overview-image-analysis", + "evidence_quote": "To use the Image Analysis APIs, you must create your Azure Vision in Foundry Tools resource in a supported region. The Image Analysis features are available in the following regions:", + "reason": "The canonical region-availability enumeration for the Image Analysis features this file covers lists 13 regions (East US, West US, West US 2, France Central, North Europe, West Europe, Sweden Central, Switzerland North, Australia East, Southeast Asia, East Asia, Korea Central, Japan East) and neither Norway East nor Norway West appears, so the claimed Norway availability is absent from the enumerating page.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-image-analysis.md", + "line": 271, + "claim": "Azure AI Vision er tilgjengelig i Norway East og Norway West", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#14", + "judge_verdict": "source_silent", + "rule": "", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/faq", + "evidence_quote": "", + "reason": "The FAQ confirms the Standard tier is named S1 and quickstarts confirm an F0 free tier exists, but no fetched or searched learn.microsoft.com page states the 0-5000 free transactions/month figure, which lives only on the JS-rendered Azure pricing page.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-image-analysis.md", + "line": 281, + "claim": "Azure AI Vision Free tier (F0): 0-5000 transaksjoner/måned gratis; Standard tier heter S1", + "disposition": "unsourced" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-ocr-processing.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 13, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#8", + "judge_verdict": "source_silent", + "rule": "", + "evidence_url": "https://learn.microsoft.com/ai-builder/prebuilt-text-recognition", + "evidence_quote": "", + "reason": "Learn confirms the AI Builder text recognition prebuilt model exists with OCR and Power Automate integration, but no Learn page states it uses Azure Vision OCR under the hood ('uses state-of-the-art optical character recognition (OCR)' names no engine), so the load-bearing engine attribution can be neither confirmed nor refuted.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-ocr-processing.md", + "line": 201, + "claim": "AI Builder tilbyr en Text Recognition prebuilt model som bruker Azure Vision OCR under panseret (Power Automate-integrasjon)", + "disposition": "unsourced" + }, + { + "id": "ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#10", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/computer-vision/ocr-data-privacy-security", + "evidence_quote": "Deletes data: The input data and results are deleted within 24 hours and not used for any other purpose.", + "reason": "The cited privacy page states input data and results are deleted within 24 hours, contradicting the claimed 48-hour retention (48 hours appears in docs only as the Operation-Location URL expiry, not data retention), so a load-bearing part is wrong.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-ocr-processing.md", + "line": 248, + "claim": "OCR data retention: input-bilder og ekstrahert tekst lagres midlertidig i 48 timer (operation-location URL), deretter slettet automatisk; ingen permanent lagring av kundedata", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#12", + "judge_verdict": "source_silent", + "rule": "", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/service-limits?view=doc-intel-4.0.0", + "evidence_quote": "", + "reason": "No learn.microsoft.com page states the monthly free allowances (5,000 transactions or 500 pages); the DI service-limits page defers 'monthly allowances' for F0 to the JS-rendered Azure pricing page, which is the expected source_silent case for pricing values.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-ocr-processing.md", + "line": 272, + "claim": "Gratisnivåer: Azure Vision v4.0 Read OCR 5000 transaksjoner/måned gratis (Standard S1) | Document Intelligence Read 500 sider/måned gratis (Standard S0)", + "disposition": "unsourced" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/content-understanding-multimodal-analysis.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 12, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#5", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/content-understanding/video/elements", + "evidence_quote": "This model may miss transitions that are visually gradual.", + "reason": "Uniform keyframe sampling, minimum one per shot, deterministic shot output, and 'first shot always starts at 0 ms' all check out, but the load-bearing claim that the service detects gradual transitions is contradicted — the page explicitly warns the model may miss visually gradual transitions.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/content-understanding-multimodal-analysis.md", + "line": 150, + "claim": "Keyframes samples uniformt fra hver camera shot, minimum én per shot, deterministisk utvalg; cameraShotTimesMs angir startpunkt per shot (første shot starter alltid ved 0 ms); detekterer abrupte og gradvise overganger", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#7", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/content-understanding/pricing-explainer", + "evidence_quote": "You pay per input unit processed: Documents: Per 1,000 pages; Audio and Video: Per minute.", + "reason": "The capability parts hold (Video Indexer celebrity/observed-people/live-stream support confirmed on insights and overview pages; CU GA lacks face identification), but the load-bearing pricing contrast is wrong: CU's GA pricing charges per-page/per-minute content extraction meters plus token-based generative charges, so 'CU prises token-basert' vs VI's minute-based pricing describes a superseded (preview-era) billing frame.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/content-understanding-multimodal-analysis.md", + "line": 242, + "claim": "Content Understanding vs Video Indexer: CU har ikke face recognition (kun face description via Limited Access) og ikke real-time analyse; Video Indexer har celebrity + custom faces, live video streaming, observed people tracking (bounding boxes); CU prises token-basert, Video Indexer page/minute-basert", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#11", + "judge_verdict": "not_grounded", + "rule": "R2", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/content-understanding/tutorial/logic-apps", + "evidence_quote": "You can use the free pricing tier (F0) to try the service, then upgrade to a paid tier for production.", + "reason": "The claim asserts no free tier exists, but a live Learn tutorial states a free pricing tier (F0) can be used to try Content Understanding, directly contradicting the non-existence claim.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/content-understanding-multimodal-analysis.md", + "line": 482, + "claim": "Free tier er ikke tilgjengelig for Content Understanding; krever betalt Azure subscription", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#12", + "judge_verdict": "not_grounded", + "rule": "R2", + "evidence_url": "https://learn.microsoft.com/en-us/python/api/overview/azure/ai-contentunderstanding-readme", + "evidence_quote": "Content Understanding operations are asynchronous long-running operations. The workflow is: 1. Begin Analysis - Start the analysis operation (returns immediately with an operation location) 2. Poll for Results - Poll the operation location until the analysis completes", + "reason": "The claim asserts a webhook-based async-completion feature exists in preview, but the canonical async-operation documentation (SDK readme, REST quickstart 'Poll the URL every 1–2 seconds until status is Succeeded') enumerates only Operation-Location polling and no CU page mentions webhooks at all — absence from the enumerating pages is evidence the feature does not exist.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/content-understanding-multimodal-analysis.md", + "line": 547, + "claim": "Webhooks for async completion er preview-feature, ikke GA (per Feb 2026)", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-custom-models.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 11, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/azure-ai-services/document-intelligence-custom-models.md#7", + "judge_verdict": "not_grounded", + "rule": "R4", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/train/composed-models?view=doc-intel-4.0.0", + "evidence_quote": "The **2024-11-30 (GA)** implementation of the `model compose` operation replaces the implicit classification from the earlier versions with an explicit classification step and adds conditional routing. ... **Assigned custom model maximum expanded to 500**.", + "reason": "The claim describes the legacy behavior the compose how-to explicitly marks as 'only applies to v3.1 and previous versions': in current v4.0 the limit is 500 (not 200) and the automatic best-match classification is replaced by an explicitly trained classifier.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-custom-models.md", + "line": 89, + "claim": "Composed models: kombiner opptil 200 custom models til én modell-ID; Document Intelligence klassifiserer dokumentet automatisk og velger best match", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/document-intelligence-custom-models.md#9", + "judge_verdict": "not_grounded", + "rule": "R2", + "evidence_url": "https://learn.microsoft.com/en-us/ai-builder/create-form-processing-model", + "evidence_quote": "**General documents**: Previously known as *unstructured*, this option is ideal for any kind of documents, especially when there's no set structure, or when the format is complex. ... This model is powerful, but has long training time. ... **Overlapping fields**: v4.0 supports overlapping fields in custom models", + "reason": "The canonical AI Builder page enumerates a 'General documents' (unstructured, long training time) model type alongside 'Fixed template documents' and lists v4.0 features (overlapping fields, signature detection, table confidence) that Document Intelligence documents as neural-only capabilities, contradicting the claim that AI Builder supports only template models.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-custom-models.md", + "line": 250, + "claim": "AI Builder custom models (Document Processing) støtter kun template models, ikke neural (per januar 2026)", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/document-intelligence-custom-models.md#11", + "judge_verdict": "not_grounded", + "rule": "", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/train/custom-lifecycle?view=doc-intel-4.0.0", + "evidence_quote": "The model is configured to expire two years after its creation for all requests utilizing a GA API to build it.", + "reason": "No Microsoft Learn page states a 90-day storage period for Document Intelligence custom models; the lifecycle page states GA-trained models expire two years after creation, a differing value for the claimed model lifetime.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-custom-models.md", + "line": 368, + "claim": "Custom models lagres i 90 dager uten kostnad", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-prebuilt-models.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 12, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#2", + "judge_verdict": "not_grounded", + "rule": "R3", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/document-intelligence/language-support/prebuilt?view=doc-intel-4.0.0", + "evidence_quote": "The following tables list the available language and locale support by model and feature ... Model ID: prebuilt-bankStatement | English (United States) en-US", + "reason": "The canonical language-support page has no uniform 27-language figure and its per-model frame contradicts one: support ranges from en-US only (bankStatement, check, contract, tax, mortgage) to ~40 languages (invoice) to 100+ (thermal receipts), so 'prebuilt models support 27 languages' matches no current enumeration.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-prebuilt-models.md", + "line": 30, + "claim": "Prebuilt-modellene støtter 27 språk", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#4", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/document-intelligence/model-overview?view=doc-intel-4.0.0", + "evidence_quote": "prebuilt-check.us | ✓ | ✓", + "reason": "Six of the seven model IDs match the page's model analysis features table, but the bank-check model ID is prebuilt-check.us, not prebuilt-check as claimed - a copy-into-code SKU string, so one load-bearing part is wrong.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-prebuilt-models.md", + "line": 40, + "claim": "Financial Services prebuilt-modeller: prebuilt-invoice | prebuilt-receipt | prebuilt-bankStatement | prebuilt-creditCard | prebuilt-check | prebuilt-contract | prebuilt-payStub.us", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#5", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/document-intelligence/model-overview?view=doc-intel-4.0.0", + "evidence_quote": "prebuilt-marriageCertificate.us | ✓ | ✓", + "reason": "The tax and ID model IDs match (prebuilt-tax.us.w2/.1098/.1099/.1040 and prebuilt-tax.us all appear), but the marriage-certificate model ID is prebuilt-marriageCertificate.us, not prebuilt-marriageCertificate as claimed - one load-bearing SKU string is wrong.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-prebuilt-models.md", + "line": 52, + "claim": "Identity & Tax prebuilt-modeller: prebuilt-idDocument | prebuilt-healthInsuranceCard.us | prebuilt-marriageCertificate | prebuilt-tax.us.w2 | prebuilt-tax.us.1098 | prebuilt-tax.us.1099 | prebuilt-tax.us.1040 | prebuilt-tax.us (unified)", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#6", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/document-intelligence/model-overview?view=doc-intel-4.0.0", + "evidence_quote": "| Closing Disclosure | Extract closing, transaction costs, and loan details. | prebuilt-mortgage.us.closingDisclosure |", + "reason": "prebuilt-mortgage.us.1003/.1004/.1005/.1008 match the page, but the fifth model ID is prebuilt-mortgage.us.closingDisclosure, not prebuilt-mortgage.us.disclosure as claimed.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-prebuilt-models.md", + "line": 65, + "claim": "US Mortgage prebuilt-modeller: prebuilt-mortgage.us.1003 | prebuilt-mortgage.us.1004 | prebuilt-mortgage.us.1005 | prebuilt-mortgage.us.1008 | prebuilt-mortgage.us.disclosure", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#7", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/document-intelligence/model-overview?view=doc-intel-4.0.0", + "evidence_quote": "All the capabilities for the general document model are available in the layout model. The general model is no longer supported.", + "reason": "prebuilt-read and prebuilt-layout descriptions are grounded, but the page states the general document model (prebuilt-document) is no longer supported in v4.0 (its 2024-11-30 column reads 'Supported in layout model'), so presenting it as a current basic model contradicts the source.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-prebuilt-models.md", + "line": 77, + "claim": "Grunnleggende modeller: prebuilt-read (OCR: tekst, linjer, ord, språkdeteksjon) | prebuilt-layout (tabeller, selection marks, seksjoner, valgfrie key-value pairs) | prebuilt-document (key-value pairs, tabeller, selection marks)", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#8", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/ai-builder/credit-management", + "evidence_quote": "AI Builder credits coming from the AI Builder add-on (1,000,000 credits per add-on) and AI Builder credits coming from licenses with seeded capacity (like Power Automate premium, which brings 5,000 credits) are gathered at the tenant level.", + "reason": "The premium-feature part holds, but the 1M-credits part is contradicted: 1,000,000 credits come only from the purchased AI Builder capacity add-on, while Power Apps/Automate licenses seed just 250-5,000 credits (and seeded credits are removed November 1, 2026); the DI v3.1 part was not confirmed either.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-prebuilt-models.md", + "line": 234, + "claim": "AI Builder bruker Document Intelligence v3.1 (ikke alltid v4.0); Premium-lisens påkrevd for AI Builder; 1M AI Builder credits inkludert i visse Power Apps/Automate-lisenser", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#10", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/document-intelligence/service-limits?view=doc-intel-4.0.0", + "evidence_quote": "Analyze transactions Per Second limit | 1 | 15 (default value) ... Max number of pages (Analysis) | 2 | 2000", + "reason": "F0 2 pages/document, S0 2,000 pages, and S0 15 TPS all match, but the F0 rate limit is stated as 1 analyze transaction per second, not '20 calls/min' as claimed (and the 500 pages/month allowance appears only on the JS-rendered pricing page), so one load-bearing part is contradicted.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-prebuilt-models.md", + "line": 391, + "claim": "Free (F0): 500 sider/måned, 2 sider per dokument, 20 calls/min | Standard (S0): 2 000 sider per dokument, 15 TPS", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#11", + "judge_verdict": "not_grounded", + "rule": "", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/document-intelligence/faq?view=doc-intel-4.0.0", + "evidence_quote": "Your data is then deleted 24 hours from the time that you submit an analyze request. If you would like the data deleted sooner, you can call the delete analyze response.", + "reason": "The documented standard retention is 24 hours (deletable sooner via the Delete Analyze Result API), not 30 days as claimed - the immediate-deletion part holds but the retention value is contradicted.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-prebuilt-models.md", + "line": 355, + "claim": "Standard 30-dagers oppbevaring av dokumenter i Document Intelligence (kan slettes umiddelbart etter prosessering)", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#12", + "judge_verdict": "not_grounded", + "rule": "", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/document-intelligence/train/custom-lifecycle?view=doc-intel-4.0.0", + "evidence_quote": "The model is configured to expire two years after its creation for all requests utilizing a GA API to build it.", + "reason": "The retraining-need part holds, but the page states a uniform two-year expiration (also two years for preview-trained models), not the claimed '12-24 months' - no 12-month expiry exists, so the stated interval contradicts the documented value.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-prebuilt-models.md", + "line": 488, + "claim": "Custom models utløper (slutter å virke) etter 12–24 måneder; krever retraining-schedule", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/language-services-custom-text-classification.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 12, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#7", + "judge_verdict": "source_silent", + "rule": "", + "evidence_url": "https://learn.microsoft.com/microsoft-365/copilot/extensibility/overview", + "evidence_quote": "", + "reason": "No learn.microsoft.com page states whether custom text classification can or cannot be integrated directly into M365 Copilot or names the Copilot Studio/Power Automate workaround; this architectural claim is neither confirmed nor contradicted by any canonical page found.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/language-services-custom-text-classification.md", + "line": 266, + "claim": "Custom Text Classification kan ikke integreres direkte i M365 Copilot (Copilot bruker forhåndstrente modeller); workaround via Copilot Studio-bot eller Power Automate-flow", + "disposition": "unsourced" + }, + { + "id": "ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#8", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/foundry/responsible-ai/language-service/data-privacy", + "evidence_quote": "Data sent in synchronous or asynchronous calls may be temporarily stored by Language for up to 48 hours only and is purged thereafter. ... To prevent this temporary storage of input data, the LoggingOptOut query parameter can be set accordingly.", + "reason": "The claim asserts no logging of runtime text and a 15-minute response cache, but the live page states input data may be temporarily stored for up to 48 hours by default (opt-out via LoggingOptOut); the no-logging and 15-minute parts are contradicted even though the LoggingOptOut parameter exists.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/language-services-custom-text-classification.md", + "line": 300, + "claim": "Tekst sendt til runtime-API logges ikke, men respons caches i 15 min (kan deaktiveres med loggingOptOut: true)", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#10", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/language-service/custom-text-classification/service-limits", + "evidence_quote": "F | Training time | One hour per month ... F | Prediction Calls | 5,000 text records per month ... Count of deployments per project (free tier) | 0 | 1", + "reason": "The 5,000 text records/month and max 1 deployment parts are confirmed, but F0 training is limited to one hour per month (Unlimited applies only to the S tier), contradicting the claimed unlimited training; one load-bearing part wrong fails the whole claim.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/language-services-custom-text-classification.md", + "line": 366, + "claim": "Free tier (F0): 5000 text records per måned for Prediction API | trening ubegrenset | maks 1 deployment per prosjekt", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#11", + "judge_verdict": "not_grounded", + "rule": "R1", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/language-service/custom-text-classification/service-limits", + "evidence_quote": "125,000 characters. You can send up to 25 documents as long as they collectively don't exceed 125,000 characters", + "reason": "The claim states a ceiling of 10 documents per request, but the live limits page states up to 25 documents per request; the stated maximum is superseded (upper-bound rule, no ratio leniency).", + "file": "skills/ms-ai-engineering/references/azure-ai-services/language-services-custom-text-classification.md", + "line": 376, + "claim": "Batch API: opptil 10 dokumenter per request", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#12", + "judge_verdict": "not_grounded", + "rule": "R2", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/language-service/custom-text-classification/language-support", + "evidence_quote": "Custom text classification supports .txt files in the following languages:", + "reason": "The canonical language-support enumeration lists 93 languages (Afrikaans through Zulu), fewer than the claimed 100+; the multilingual option only covers these same supported languages, so the asserted floor is not met.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/language-services-custom-text-classification.md", + "line": 401, + "claim": "Custom Classification støtter 100+ språk (multilingual-funksjonalitet)", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/language-services-question-answering.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 15, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/azure-ai-services/language-services-question-answering.md#12", + "judge_verdict": "source_silent", + "rule": "", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/language-service/question-answering/concepts/azure-resources", + "evidence_quote": "", + "reason": "The cited page only states that CQA's management and prediction services are colocated in the same region; neither it nor any Learn page found via search states (or contradicts) that the Azure AI Search resource must be in the same region as the Language resource or that cross-region replication is disallowed.", + "file": "skills/ms-ai-engineering/references/azure-ai-services/language-services-question-answering.md", + "line": 388, + "claim": "Cross-region replication er ikke tillatt i CQA; Azure AI Search må være i samme region som Language resource", + "disposition": "unsourced" + }, + { + "id": "ms-ai-engineering/azure-ai-services/language-services-question-answering.md#14", + "judge_verdict": "source_silent", + "rule": "", + "evidence_url": "", + "evidence_quote": "", + "reason": "The F0 5,000-text-records/month allowance and S-tier unlimited hosted calls are Azure pricing-page (JS-rendered) values; no learn.microsoft.com page found states them (the CQA limits page only notes F0 upload is limited to three files).", + "file": "skills/ms-ai-engineering/references/azure-ai-services/language-services-question-answering.md", + "line": 465, + "claim": "Language resource med CQA: Free (F0) 5000 text records/måned | Standard (S) ubegrensede hosted calls", + "disposition": "unsourced" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/mlops-genaiops/prompt-flow-production-deployment.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "pass", + "claim_count": 13, + "verified": "2026-07-18", + "verified_by": "judge-v3.1", + "flags": [] + }, + { + "file": "skills/ms-ai-engineering/references/rag-architecture/citation-tracking.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 10, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/rag-architecture/citation-tracking.md#3", + "judge_verdict": "not_grounded", + "rule": "", + "evidence_url": "https://learn.microsoft.com/azure/foundry-classic/openai/references/on-your-data", + "evidence_quote": "Azure OpenAI On Your Data is deprecated and approaching retirement. The service will be retired on October 14, 2026", + "reason": "The claim asserts 'On Your Data' with citation annotations is GA, but the live page states the service is deprecated and retires October 14, 2026 with Microsoft recommending migration to Foundry Agent Service — a superseded status a reader would act on.", + "file": "skills/ms-ai-engineering/references/rag-architecture/citation-tracking.md", + "line": 26, + "claim": "Classic RAG (Azure AI Search + Azure OpenAI, 'On Your Data' med citation annotations) er GA", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/rag-architecture/citation-tracking.md#6", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/evaluation-evaluators/rag-evaluators", + "evidence_quote": "Groundedness Pro uses the Azure AI Content Safety service and returns a boolean result instead of a numeric score", + "reason": "The AACS-delivery part holds, but the load-bearing 'threshold-based scoring' part is contradicted: the live page reserves 'Pass/Fail based on threshold (1-5 scale)' for the other evaluators and states Groundedness Pro returns a boolean True/False instead of a numeric score.", + "file": "skills/ms-ai-engineering/references/rag-architecture/citation-tracking.md", + "line": 213, + "claim": "Groundedness Pro: threshold-basert scoring, leveres av Azure AI Content Safety (per evaluerings-kall)", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-context-windows.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 11, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#10", + "judge_verdict": "not_grounded", + "rule": "R3", + "evidence_url": "https://learn.microsoft.com/en-us/azure/foundry/openai/quotas-limits", + "evidence_quote": "Quota values for global batch are represented in terms of enqueued tokens. ... Global batch: | Model | Enterprise and MCA-E | Default | Monthly credit card-based subscriptions | MSDN subscriptions | Azure for Students, free trials | `gpt-4.1` | 5B | 200M | 50M | 90K | N/A | ... `gpt-4o` | 5B | 200M | 50M | 90K | N/A", + "reason": "The live page organizes batch quota by subscription/agreement type (Enterprise/MCA-E, Default, credit card, MSDN) as enqueued tokens, not per Quota Tier per month, and global batch for gpt-4.1 and gpt-4o is 5B/200M/50M — the claim's ~500M/~30M figures match only the separate Data zone batch table, so both the organizing frame and the headline values are wrong.", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-context-windows.md", + "line": 339, + "claim": "Azure OpenAI Batch quota (per Quota Tier): gpt-4.1 ~500M tokens/month (upper tier), ~30M (lower tier); gpt-4o ~500M (upper), ~30M (lower)", + "disposition": "outdated" + }, + { + "id": "ms-ai-engineering/rag-architecture/rag-context-windows.md#11", + "judge_verdict": "source_silent", + "rule": "", + "evidence_url": "https://learn.microsoft.com/en-us/azure/foundry-classic/how-to/flow-develop", + "evidence_quote": "", + "reason": "No learn.microsoft.com page states that Prompt Flow shows an estimated token count per node in the flow: the develop-flow page documents a trace view with duration and token cost of the flow (per-node detail is duration), and per-node token_consumption metrics exist only in Application Insights for deployed flows, so the claimed in-flow per-node estimate can neither be confirmed nor refuted.", + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-context-windows.md", + "line": 273, + "claim": "Prompt Flow (Microsoft Foundry) shows estimated token count per node in the flow", + "disposition": "unsourced" + } + ] + }, + { + "file": "skills/ms-ai-governance/references/responsible-ai/content-safety-implementation.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 12, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-governance/responsible-ai/content-safety-implementation.md#1", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/content-safety/whats-new", + "evidence_quote": "Protected material detection for code (preview)", + "reason": "The claim marks Protected Material Code as GA, but What's new and the current quickstart both title it '(preview)' with no GA announcement (the Aug 2024 GA covered only Prompt Shields and Protected Material for text), so one load-bearing status in the bundle is wrong while the other statuses match.", + "file": "skills/ms-ai-governance/references/responsible-ai/content-safety-implementation.md", + "line": 37, + "claim": "Content Safety feature statuses: Analyze Text GA | Analyze Image GA | Prompt Shields GA | Groundedness Detection Preview | Protected Material Text GA | Protected Material Code GA | Custom Categories (Standard) Preview | Custom Categories (Rapid) Preview | Blocklists GA", + "disposition": "outdated" + }, + { + "id": "ms-ai-governance/responsible-ai/content-safety-implementation.md#2", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/content-safety/overview", + "evidence_quote": "Maximum length for grounding sources: 55,000 characters (per API call). Maximum text and query length: 7,500 characters.", + "reason": "The claim states 'Groundedness query+sources max 55K chars', but the live page gives 55,000 characters to grounding sources only and caps text/query separately at 7,500 characters, so that load-bearing limit is misstated even though the other limits (Analyze Text 10K, image 4MB JPEG/PNG/GIF/BMP/TIFF/WEBP, Prompt Shields 10K, PM-Text min 110) match.", + "file": "skills/ms-ai-governance/references/responsible-ai/content-safety-implementation.md", + "line": 37, + "claim": "Content Safety input limits: Analyze Text max 10K chars | Analyze Image JPEG/PNG/GIF/BMP/TIFF/WEBP max 4MB | Prompt Shields text max 10K chars | Groundedness query+sources max 55K chars | Protected Material Text min 110 chars", + "disposition": "outdated" + }, + { + "id": "ms-ai-governance/responsible-ai/content-safety-implementation.md#6", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/microsoft-copilot-studio/knowledge-copilot-studio", + "evidence_quote": "The moderation levels range from **Lowest** to **Highest**. The lowest level generates the most answers, but they might contain harmful content. The highest level of content moderation generates fewer answers, and applies a stricter filter to restrict harmful content. The default moderation level is **High**.", + "reason": "The claim says moderation 'uses Azure OpenAI deployment settings' and that custom blocklists can be enabled in Agent Settings, but the live docs show content moderation is configured inside Copilot Studio itself (agent-, topic-, and prompt-level Lowest-Highest levels) and the Agent settings enumeration has a Moderation level dropdown with no custom-blocklist option, so two load-bearing parts are wrong (only the no-per-category-severity part holds).", + "file": "skills/ms-ai-governance/references/responsible-ai/content-safety-implementation.md", + "line": 239, + "claim": "Copilot Studio cannot (per feb 2026) configure severity levels per category - uses Azure OpenAI deployment settings; custom blocklists can be enabled in Agent Settings", + "disposition": "outdated" + }, + { + "id": "ms-ai-governance/responsible-ai/content-safety-implementation.md#7", + "judge_verdict": "not_grounded", + "rule": "R3", + "evidence_url": "https://learn.microsoft.com/ai-builder/prebuilt-azure-openai", + "evidence_quote": "This feature is deprecated and isn't visible anymore.", + "reason": "AI Builder's Text generation model is deprecated and replaced by prompt builder, where per the prompts FAQ 'Makers can configure the content moderation level for harmful content only' - so the claim's frame (no configuration options, implicit Medium+High default) describes a replaced world and its 'no configuration' part is contradicted for the current mechanism.", + "file": "skills/ms-ai-governance/references/responsible-ai/content-safety-implementation.md", + "line": 243, + "claim": "AI Builder Text generation uses Azure OpenAI with content filtering enabled by default and no configuration options (default Medium+High block)", + "disposition": "outdated" + }, + { + "id": "ms-ai-governance/responsible-ai/content-safety-implementation.md#8", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/microsoft-365/copilot/harmful-content-protection-copilot-chat", + "evidence_quote": "For these special use cases, your organization's Microsoft 365 Apps administrator can enable a specific group of users to adjust harmful content protection settings in their Copilot Chat experiences.", + "reason": "The claim's load-bearing 'not customer-configurable' is contradicted by the live page: admins can policy-enable users to toggle harmful content protection off in Copilot Chat (and the privacy doc says Microsoft 'offers certain content filtering controls for admins and users'), even though severity-level sliders indeed do not exist.", + "file": "skills/ms-ai-governance/references/responsible-ai/content-safety-implementation.md", + "line": 259, + "claim": "Microsoft 365 Copilot has its own content filtering policies that are not customer-configurable (severity levels cannot be adjusted; Microsoft-managed)", + "disposition": "outdated" + }, + { + "id": "ms-ai-governance/responsible-ai/content-safety-implementation.md#9", + "judge_verdict": "not_grounded", + "rule": "R2", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-services/content-safety/overview", + "evidence_quote": "To use the Content Safety APIs, you must create your Azure AI Content Safety resource in a supported region. Currently, the Content Safety features are available in the following Azure regions with different API versions:", + "reason": "The canonical region-availability table on the cited page lists West Europe but contains no Norway East row for any Content Safety API, and no Learn page found states Content Safety availability in Norway East (even via Azure OpenAI), so the Norway East part fails the enumeration check; the in-region/no-US-transfer part alone is supported by the data-privacy page.", + "file": "skills/ms-ai-governance/references/responsible-ai/content-safety-implementation.md", + "line": 275, + "claim": "Azure AI Content Safety available in West Europe and Norway East (via Azure OpenAI); models run in EU (no data transfer to USA)", + "disposition": "outdated" + }, + { + "id": "ms-ai-governance/responsible-ai/content-safety-implementation.md#10", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/language-service/personally-identifiable-information/concepts/entity-categories", + "evidence_quote": "To retrieve this entity type, specify **NOIdentityNumber** in the **piiCategories** request parameter. If detected, the entity appears in the **PII** response payload.", + "reason": "The PII filter page links this entity-categories list as 'the complete list of supported personal data entity types' and lists 'National ID numbers (50+ countries)' including a dedicated Norway Identity Number (NOIdentityNumber) type, contradicting the claim's load-bearing assertion that the Norwegian national ID format is not officially supported (the detection-in-completions and block/redacted_text parts do hold).", + "file": "skills/ms-ai-governance/references/responsible-ai/content-safety-implementation.md", + "line": 270, + "claim": "Azure AI Content Safety has PII detection for completions (names, addresses; Norwegian national ID format not officially supported), configurable to block or mask PII in LLM output", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-security/references/ai-security-engineering/ai-prompt-shield-network.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 12, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#1", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/entra/fundamentals/whats-new-archive", + "evidence_quote": "General Availability - Protect enterprise GenAI applications with Prompt Injection Protection", + "reason": "The Content Safety Prompt Shields GA part holds, but the Entra whats-new archive (November 2025) announces General Availability of the AI Gateway prompt protection capability (now named Prompt Injection Protection), contradicting the claim's Preview status.", + "file": "skills/ms-ai-security/references/ai-security-engineering/ai-prompt-shield-network.md", + "line": 6, + "claim": "Two separate products: Azure AI Content Safety Prompt Shields is GA; AI Gateway Prompt Shield via Global Secure Access is Preview", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#3", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/content-safety/overview", + "evidence_quote": "Prompt Shields API: Maximum prompt length: 10K characters. Up to five documents with a total of 10K characters.", + "reason": "Prompt limit 10K and max 5 documents hold, but the claim's 'each document max 10 000 chars' contradicts the source, which caps all documents at 10K characters combined, not per document.", + "file": "skills/ms-ai-security/references/ai-security-engineering/ai-prompt-shield-network.md", + "line": 139, + "claim": "Prompt Shields input limits: userPrompt max 10 000 chars | documents array max 5 documents per request | each document max 10 000 chars", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#4", + "judge_verdict": "not_grounded", + "rule": "R2", + "evidence_url": "https://learn.microsoft.com/entra/global-secure-access/how-to-ai-prompt-injection-protection", + "evidence_quote": "Prompt Injection Protection is preconfigured with custom extractors for the following models: ChatGPT, Claude, Cohere, Deepseek, Gemini, Grok, Meta AI, Mistral, Perplexity, Pi, and Qwen.", + "reason": "The canonical enumeration does not include 'Microsoft Copilot' and lists 'Meta AI' rather than 'Meta Llama' (and now also includes Deepseek, Gemini, Perplexity), so the claim's stated extractor list disagrees with the current page.", + "file": "skills/ms-ai-security/references/ai-security-engineering/ai-prompt-shield-network.md", + "line": 180, + "claim": "AI Gateway Prompt Shield has pre-configured extractors for: Microsoft Copilot, OpenAI ChatGPT, Anthropic Claude, Meta Llama, xAI Grok, Mistral, Cohere, Inflection Pi, Alibaba Qwen, plus custom JSON-based LLMs (custom URL + JSON path)", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#5", + "judge_verdict": "not_grounded", + "rule": "R1", + "evidence_url": "https://learn.microsoft.com/entra/global-secure-access/how-to-ai-prompt-injection-protection", + "evidence_quote": "Prompt Injection Protection supports prompts up to 64,000 characters. Anything longer is truncated.", + "reason": "Text-only and JSON-only limitations still hold, but the claimed 10 000-char ceiling is superseded — the live page states 64,000 characters, and Gemini (the claim's example of an unsupported app) is now a preconfigured extractor.", + "file": "skills/ms-ai-security/references/ai-security-engineering/ai-prompt-shield-network.md", + "line": 193, + "claim": "AI Gateway Prompt Shield limitations: text prompts only (no files); JSON-based GenAI apps only (not URL-encoded, e.g. Gemini); max 10 000 chars per prompt (longer prompts truncated)", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#11", + "judge_verdict": "not_grounded", + "rule": "R7", + "evidence_url": "https://learn.microsoft.com/azure/ai-services/content-safety/quickstart-groundedness", + "evidence_quote": "The current version is: api-version=2024-09-15-preview. Example: /contentsafety/text:detectGroundedness?api-version=2024-09-15-preview", + "reason": "The endpoint path matches, but the load-bearing api-version string is wrong — the canonical quickstart states 2024-09-15-preview as the current version, not 2024-09-01.", + "file": "skills/ms-ai-security/references/ai-security-engineering/ai-prompt-shield-network.md", + "line": 309, + "claim": "Groundedness endpoint: POST {endpoint}/contentsafety/text:detectGroundedness?api-version=2024-09-01", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#12", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/entra/fundamentals/whats-new-archive", + "evidence_quote": "General Availability - Protect enterprise GenAI applications with Prompt Injection Protection", + "reason": "The license-inclusion part is supported (the feature's prerequisites require a Microsoft Entra Internet Access license), but the stated '(Preview)' status is contradicted — the November 2025 Entra release notes announce General Availability.", + "file": "skills/ms-ai-security/references/ai-security-engineering/ai-prompt-shield-network.md", + "line": 428, + "claim": "AI Gateway Prompt Shield (Preview) is included in the Microsoft Entra Internet Access license (licensed per user/month)", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-security/references/ai-security-engineering/entra-agent-id-zero-trust.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 13, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#1", + "judge_verdict": "not_grounded", + "rule": "R4", + "evidence_url": "https://learn.microsoft.com/entra/fundamentals/whats-new", + "evidence_quote": "The Microsoft Entra Agent ID platform is now generally available.", + "reason": "The claim asserts Public Preview status, but the April 2026 release notes announce general availability and the live product page now states 'Agent ID is available for all Microsoft Entra customers' with no preview disclaimer.", + "file": "skills/ms-ai-security/references/ai-security-engineering/entra-agent-id-zero-trust.md", + "line": 5, + "claim": "Microsoft Entra Agent ID har status Public Preview (utvidet etter Ignite 2025)", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#3", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/microsoft-copilot-studio/admin-use-entra-agent-identities", + "evidence_quote": "Starting July 2026, all new agents must have Microsoft Entra Agent IDs, and you can no longer opt out of automatic agent identity creation.", + "reason": "The blueprint name and Blueprint ID 25664c89-cea5-4ab6-b924-a54fd8a19ae0 match, but the load-bearing 'preview, activated per environment in Power Platform Admin Center' framing is superseded - creation is now automatic and mandatory with the per-environment opt-out removed.", + "file": "skills/ms-ai-security/references/ai-security-engineering/entra-agent-id-zero-trust.md", + "line": 20, + "claim": "Copilot Studio (preview, aktiveres per miljø i Power Platform Admin Center) oppretter automatisk Entra Agent ID for hver ny agent, knyttet til Microsoft Copilot Studio agent identity blueprint med Blueprint ID 25664c89-cea5-4ab6-b924-a54fd8a19ae0", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#10", + "judge_verdict": "not_grounded", + "rule": "R4", + "evidence_url": "https://learn.microsoft.com/en-us/microsoft-copilot-studio/admin-use-entra-agent-identities", + "evidence_quote": "Previously, you could opt out of Entra Agent ID at the environment level. Starting July 2026, all new agents must have Microsoft Entra Agent IDs, and you can no longer opt out of automatic agent identity creation.", + "reason": "The claim describes the opt-out as currently existing with mandatory enforcement coming 'in the future', but the current page states the opt-out is already removed and Agent IDs are mandatory for all new agents as of July 2026.", + "file": "skills/ms-ai-security/references/ai-security-engineering/entra-agent-id-zero-trust.md", + "line": 322, + "claim": "Opt-out fra Entra Agent Identity per miljø i Copilot Studio er midlertidig — Microsoft vil gjøre det obligatorisk for alle nye agenter i fremtiden", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#11", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/entra/fundamentals/whats-new", + "evidence_quote": "The Microsoft Entra Agent ID platform is now generally available.", + "reason": "The status table's load-bearing 'Public Preview (Frontier program)' frame is superseded: the Agent ID platform reached GA in April 2026, the live product page lists Agent 365/E5/E7/P1-P2 licensing with no Frontier requirement, and the Entra Agent Registry blades were retired May 1, 2026 in the consolidation into Microsoft Agent 365 - even though the AI Prompt Shield, App Service/Functions, and Teams Developer Portal items individually check out.", + "file": "skills/ms-ai-security/references/ai-security-engineering/entra-agent-id-zero-trust.md", + "line": 439, + "claim": "Statustabell: Entra Agent ID kjerne/Agent Registry/Identity Protection for agenter/Global Secure Access for agenter = Public Preview (Frontier-program); Foundry-integrasjon Public Preview (alle Foundry-brukere); Conditional Access for agenter Public Preview; AI Prompt Shield nytt via Entra Internet Access; App Service/Azure Functions agent identity og Teams Developer Portal agent blueprints nye", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#12", + "judge_verdict": "not_grounded", + "rule": "R3", + "evidence_url": "https://learn.microsoft.com/entra/agent-id/identity-professional/microsoft-entra-agent-identities-for-ai-agents", + "evidence_quote": "Extending Microsoft Entra security features to agents requires Microsoft 365 E7 (includes Agent 365 and Microsoft Entra Suite) or Microsoft 365 E5 paired with a Microsoft Agent 365 license.", + "reason": "The Frontier-program + M365 Copilot licensing frame has been replaced: the live page states full Agent ID security functionality requires Agent 365/E5/E7 (or P1/P2 standalone options) with no mention of Frontier enrollment or the Copilot Frontier admin-center toggle, which now applies only to pre-release Agent 365 features.", + "file": "skills/ms-ai-security/references/ai-security-engineering/entra-agent-id-zero-trust.md", + "line": 452, + "claim": "Full Entra Agent ID-funksjonalitet krever deltakelse i Microsoft Frontier-programmet og M365 Copilot-lisens (aktiveres via M365 admin center → Copilot → Settings → User access → Copilot Frontier); Foundry-integrert agentidentitet er tilgjengelig for alle Foundry-brukere uten Frontier", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-security/references/ai-security-engineering/zero-trust-ai-services.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 11, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#2", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/managed-identity-best-practice-recommendations", + "evidence_quote": "This means that it can take several hours for changes to a managed identity's group or role membership to take effect.", + "reason": "The 24-hour token cache and hours-long propagation parts are confirmed, but the page explicitly treats group AND app-role membership identically (both are claims in the cached token, same several-hours delay, and it recommends user-assigned MI with direct permissions for fast changes), contradicting the load-bearing part that App Roles propagate faster than groups.", + "file": "skills/ms-ai-security/references/ai-security-engineering/zero-trust-ai-services.md", + "line": 128, + "claim": "Managed Identity tokens are cached up to 24 hours; group/role membership changes can take hours to propagate; App Roles propagate faster than groups", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#3", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/navigate-from-classic", + "evidence_quote": "Microsoft's AI Platform has evolved from Azure AI Studio -> Azure AI Foundry -> to Microsoft Foundry (current). Similarly, our AI services portfolio evolved with the platform from Azure Cognitive Services -> Azure AI Services -> to Foundry Tools (current).", + "reason": "The avoid-API-keys recommendation, kind=AIServices, and DefaultAzureCredential pattern are grounded, but the naming part 'formerly Foundry Tools' is contradicted: Foundry Tools is the CURRENT name of the AI services portfolio (formerly Azure AI Services), not a former name of the Microsoft Foundry resource (whose lineage is Azure AI Studio -> Azure AI Foundry -> Microsoft Foundry).", + "file": "skills/ms-ai-security/references/ai-security-engineering/zero-trust-ai-services.md", + "line": 130, + "claim": "Microsoft now explicitly recommends avoiding API keys for Azure AI Services in production; Microsoft Foundry resource (formerly 'Foundry Tools', kind=AIServices) uses the same DefaultAzureCredential pattern across all AI services", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#4", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/entra/identity/conditional-access/concept-continuous-access-evaluation", + "evidence_quote": "The goal for critical event evaluation is for response to be near real time, but latency of up to 15 minutes might be observed because of event propagation time; however, IP locations policy enforcement is instant.", + "reason": "The 28-hour CAE token lifetime and 1-hour standard token parts are confirmed on the page (and the MI 24-hour cache on the MI best-practices page), but the load-bearing part 'critical events revoke in seconds' is contradicted: the page states near real time with up to 15 minutes latency and 'within minutes after a critical event', not seconds.", + "file": "skills/ms-ai-security/references/ai-security-engineering/zero-trust-ai-services.md", + "line": 149, + "claim": "CAE token lifetime: up to 28 hours (vs standard access token 1 hour, Managed Identity refresh token 24 hours); critical events revoke in seconds", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#5", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/entra/identity/conditional-access/concept-continuous-access-evaluation-strict-enforcement", + "evidence_quote": "Standard (Default) | Suitable for all topologies | A short-lived token is issued only if Microsoft Entra ID detects an allowed IP address. Otherwise, access is blocked", + "reason": "The move to Conditional Access session controls is confirmed ('The CAE setting moved to Conditional Access.'), but the option set is wrong: the documented options are Disable (session control) and Strictly enforce location policies, with the default enforcement mode named 'Standard (Default)' - no option named 'Basic' exists on the session-control or strict-enforcement pages.", + "file": "skills/ms-ai-security/references/ai-security-engineering/zero-trust-ai-services.md", + "line": 152, + "claim": "From 2025 CAE is configured via Conditional Access policies (Session controls) instead of a separate toggle; options Disabled, Basic (default), Strict", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#6", + "judge_verdict": "not_grounded", + "rule": "R2", + "evidence_url": "https://learn.microsoft.com/en-us/entra/identity/conditional-access/concept-continuous-access-evaluation", + "evidence_quote": "Continuous access evaluation is also available in Azure Government tenants (GCC High and DOD) for Exchange Online.", + "reason": "Preview status, per-policy activation, and near-realtime blocking are confirmed on the strict-enforcement page, but 'now supports Azure Government clouds' is absent from both canonical pages - the only Government-availability statement covers CAE generally and only for Exchange Online, so the claimed strict-enforcement Government support is not documented.", + "file": "skills/ms-ai-security/references/ai-security-engineering/zero-trust-ai-services.md", + "line": 154, + "claim": "Strict Location Enforcement is Preview, activated per Conditional Access policy, blocks tokens used outside approved network locations in near-realtime, and now supports Azure Government clouds", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#9", + "judge_verdict": "not_grounded", + "rule": "", + "evidence_url": "https://learn.microsoft.com/power-platform/admin/vnet-support-overview", + "evidence_quote": "| Dataverse | Dataverse plug-ins | Generally available |", + "reason": "The cited vnet-data-gateway page returns 404; the live Virtual Network support overview states Dataverse (plug-ins) Virtual Network support is Generally available, contradicting the claimed 'Private Preview (Q1 2026)' status.", + "file": "skills/ms-ai-security/references/ai-security-engineering/zero-trust-ai-services.md", + "line": 606, + "claim": "Virtual Network Integration for Dataverse is in Private Preview (Q1 2026)", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-security/references/cost-optimization/azure-ai-foundry-cost-governance.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 9, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#7", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/foundry/how-to/navigate-from-classic", + "evidence_quote": "| Resource type | Azure OpenAI + Hub | Foundry Resource | Single `AIServices` kind with child projects. | | AI services | Azure AI Services | Foundry Tools | Speech, Vision, Language, Content Safety, Content Understanding. |", + "reason": "The kind-AIServices-to-Microsoft-Foundry-resource part and projects-as-folders part are grounded, but the terminology mapping shows 'Foundry Tools' is the CURRENT name for the constituent AI services (formerly Azure AI Services), coexisting with the Foundry resource — not the predecessor name the Foundry resource was renamed from — so one load-bearing lineage part is contradicted.", + "file": "skills/ms-ai-security/references/cost-optimization/azure-ai-foundry-cost-governance.md", + "line": 121, + "claim": "The underlying Azure resource (API kind AIServices) is renamed Microsoft Foundry resource (successor/renaming of 'Foundry Tools'); projects function as folders grouping work under one resource", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-security/references/cost-optimization/batch-processing-cost-reduction.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 11, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-security/cost-optimization/batch-processing-cost-reduction.md#2", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/deployment-types", + "evidence_quote": "DataZone types: The service processes data only within the Microsoft-specified data zone (US, EU, or Asia Pacific (APAC)). Standard/Regional types: Processed in the deployment region", + "reason": "The two type names Global-Batch and Data Zone Batch are confirmed, but the load-bearing gloss 'regionsbasert' is contradicted: the page's taxonomy explicitly separates data-zone processing (US/EU/APAC zone spanning many regions) from regional processing (single deployment region), so calling Data Zone Batch region-based misstates its data-residency category.", + "file": "skills/ms-ai-security/references/cost-optimization/batch-processing-cost-reduction.md", + "line": 29, + "claim": "To batch deployment-typer: Global-Batch (globalt distribuert) | Data Zone Batch (regionsbasert)", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 17, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#1", + "judge_verdict": "not_grounded", + "rule": "", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure?view=foundry-classic", + "evidence_quote": "- 1,047,576 - 300,000 (standard deployments) - 128,000 (provisioned managed and batch deployments)", + "reason": "The page states 300,000 for standard deployments and 128,000 for provisioned managed AND batch deployments, while the claim swaps them (128K for standard/provisioned, 300K for batch).", + "file": "skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md", + "line": 56, + "claim": "gpt-4.1 kontekstvindu: 1047576 tokens (full), 128000 tokens (standard og provisioned deployments), 300000 tokens (batch deployments)", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#5", + "judge_verdict": "not_grounded", + "rule": "R4", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning", + "evidence_quote": "Access is no longer restricted for this model.", + "reason": "The live availability table states this for both gpt-5 and gpt-5-codex, so the claimed registration/approval requirement is superseded (the mini/nano/chat 'No access request needed' half still matches, but the load-bearing requirement part is now wrong).", + "file": "skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md", + "line": 83, + "claim": "gpt-5 og gpt-5-codex krever registrering og godkjenning; gpt-5-mini, gpt-5-nano, gpt-5-chat har ingen registreringskrav", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#9", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/microsoft-copilot-studio/prompt-model-settings", + "evidence_quote": "| Claude Sonnet 4.6 | Standard rate | External model from Anthropic. Context allowed up to 200K tokens. | General |", + "reason": "The current model table lists Claude Sonnet 4.6 (Standard) and Claude Opus 4.6 (Premium) instead of the claimed 4.5 versions and contains no o3 row at all, so multiple load-bearing entries in the claimed rate lineup are superseded or absent.", + "file": "skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md", + "line": 203, + "claim": "Copilot Credits-takstnivåer: gpt-4.1-mini Basic | gpt-4.1 Standard | gpt-5-chat (preview) Standard | gpt-5-reasoning (preview) Premium | o3 Premium | Claude Sonnet 4.5 (experimental) Standard | Claude Opus 4.5 (experimental) Premium", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#11", + "judge_verdict": "not_grounded", + "rule": "R2", + "evidence_url": "https://learn.microsoft.com/microsoft-copilot-studio/prompt-model-settings", + "evidence_quote": "| Claude Sonnet 4.6 | Standard rate | External model from Anthropic. Context allowed up to 200K tokens. | General | | Claude Opus 4.6 | Premium rate | External model from Anthropic. Context allowed up to 200K tokens. | Deep |", + "reason": "The canonical model enumeration lists Claude Sonnet 4.6 and Claude Opus 4.6 (200K); the claimed Sonnet 4.5/Opus 4.5 are absent, i.e. superseded by 4.6 versions.", + "file": "skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md", + "line": 217, + "claim": "Claude Sonnet 4.5 og Claude Opus 4.5 er tilgjengelig i Copilot Studio (experimental, 200K kontekstvindu)", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#14", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure-region-availability?pivots=standard", + "evidence_quote": "| gpt-5.4 | 2026-03-05 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |", + "reason": "The Data Zone Standard Europe table now shows gpt-5.4, gpt-5.5, and gpt-5.6-sol/terra/luna all available in norwayeast, so the load-bearing 'eneste GPT-5 ... er gpt-5.5' part is superseded, even though the no-Regional/PTU parts (gpt-4.1, o-series, GPT-5 family absent in Norway East) still verify.", + "file": "skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md", + "line": 393, + "claim": "Norway East: kun gpt-4o/gpt-4o-mini tilgjengelig som Norge-resident (Standard/Regional PTU); gpt-4.1, o-serien (o3/o4-mini/o3-mini/o1) og GPT-5-familien finnes ikke som Regional/PTU i Norway East; eneste GPT-5 med EU-residens i Norway East er gpt-5.5 via Data Zone Standard", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#16", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning", + "evidence_quote": "Access is no longer restricted for this model.", + "reason": "The live availability table states this for gpt-5, gpt-5-codex, and gpt-5-pro, contradicting the claimed aka.ms/oai/gpt5access approval requirement and the MCA-E/Default-only restriction for gpt-5-pro; the gpt-5-chat 'Preview (2 versions)' part matches, but the load-bearing access-status parts are superseded.", + "file": "skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md", + "line": 468, + "claim": "GPT-5-tilgjengelighet: gpt-5 GA (begrenset, krever godkjenning aka.ms/oai/gpt5access) | gpt-5-mini GA | gpt-5-nano GA | gpt-5-chat Preview (2 versjoner) | gpt-5-codex GA (begrenset) | gpt-5-pro GA (begrenset, kun MCA-E/Default-abonnementer)", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#17", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/ai-builder/credit-management", + "evidence_quote": "| Power Apps Premium | 500 | Maximum = 1,000,000 AI Builder credits per tenant. | ... | Power Automate Premium | 5,000 | Maximum = 1,000,000 AI Builder credits per tenant. |", + "reason": "The 500-credit figure applies only to Power Apps Premium; other premium Power Platform plans seed 250 to 5,000 AI Builder credits, and the page adds that these seeded credits are removed on November 1, 2026, so the blanket '500 credits/bruker/mnd i premium-planer' is contradicted even though the gpt-4.1-mini Basic default is correct.", + "file": "skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md", + "line": 443, + "claim": "AI Builder: prompt builder credits inkludert i premium Power Platform-planer (500 credits/bruker/mnd); default modell gpt-4.1-mini (Basic rate)", + "disposition": "outdated" + } + ] + }, + { + "file": "skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md", + "batch": "R7.1", + "judged_at": "2026-07-18", + "per_file_verdict": "flagged", + "claim_count": 16, + "verified": null, + "verified_by": null, + "flags": [ + { + "id": "ms-ai-security/cost-optimization/token-counting-optimization.md#1", + "judge_verdict": "source_silent", + "rule": "", + "evidence_url": "https://learn.microsoft.com/dotnet/ai/conceptual/understanding-tokens", + "evidence_quote": "", + "reason": "No learn.microsoft.com page states that o200k_base is the default tiktoken encoding for gpt-4o/o1/o3 (the cited source is GitHub, and two targeted searches found only cl100k_base/BPE mentions on Learn), so the claim can be neither confirmed nor refuted from Microsoft Learn.", + "file": "skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md", + "line": 53, + "claim": "o200k_base er default tiktoken-encoding for gpt-4o, o1, o3", + "disposition": "unsourced" + }, + { + "id": "ms-ai-security/cost-optimization/token-counting-optimization.md#6", + "judge_verdict": "source_silent", + "rule": "", + "evidence_url": "https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/prompt-caching", + "evidence_quote": "", + "reason": "The page confirms cache reads are discounted on Standard and 'up to 100% discount on input tokens' for Provisioned, but the load-bearing 50% figure for Standard is never stated on any Learn page - it lives on the JS-rendered Azure pricing page, so it can be neither confirmed nor refuted.", + "file": "skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md", + "line": 148, + "claim": "Cached tokens: 50% rabatt på Standard deployment; opptil 100% rabatt på Provisioned (inkludert i PTU-pris)", + "disposition": "unsourced" + }, + { + "id": "ms-ai-security/cost-optimization/token-counting-optimization.md#8", + "judge_verdict": "not_grounded", + "rule": "", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models", + "evidence_quote": "| `text-embedding-ada-002` (version 2) | 8,192 | 1,536 | Sep 2021 | ... | `text-embedding-3-large` | 8,192 | 3,072 | Sep 2021 | | `text-embedding-3-small` | 8,192 | 1,536 | Sep 2021 |", + "reason": "The current canonical models table states max request of 8,192 tokens for ada-002 v2, 3-small, and 3-large (the REST reference likewise says '8192 tokens for all embedding models'), which differs from the claimed 8191.", + "file": "skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md", + "line": 263, + "claim": "Token-limit per chunk for embeddings: text-embedding-ada-002 8191 tokens | text-embedding-3-small/large 8191 tokens", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/cost-optimization/token-counting-optimization.md#9", + "judge_verdict": "not_grounded", + "rule": "", + "evidence_url": "https://learn.microsoft.com/azure/ai-foundry/openai/concepts/models#fine-tuning-models", + "evidence_quote": "| `gpt-4o-mini` (2024-07-18) | North Central US Sweden Central | - | Input: 128,000 Output: 16,384 Training example context length: 65,536 | Oct 2023 |", + "reason": "The canonical fine-tuning models table states a training example context length of 65,536 tokens for gpt-4o-mini, contradicting the claimed 64,536 (a figure that appears only in the tutorial and is superseded by the spec table); the 128,000 input limit part does hold.", + "file": "skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md", + "line": 327, + "claim": "gpt-4o-mini fine-tuning: training example max 64536 tokens, input limit 128000 tokens", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/cost-optimization/token-counting-optimization.md#14", + "judge_verdict": "not_grounded", + "rule": "R8", + "evidence_url": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models", + "evidence_quote": "| `gpt-4.1` (2025-04-14) | ... | - 1,047,576 - 300,000 (standard deployments) - 128,000 (provisioned managed and batch deployments) | 32,768 | May 31, 2024 |", + "reason": "The gpt-4.1 context window is 1,047,576 tokens (~1M) on the live models page, not the claimed 128K (128,000 is only the provisioned/batch deployment cap), so one load-bearing part is wrong even though gpt-5/gpt-5-mini 400K, gpt-4o/gpt-4o-mini 128K, and o3-mini 200K all match.", + "file": "skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md", + "line": 480, + "claim": "Context windows: gpt-5 400K | gpt-5-mini 400K | gpt-4o 128K | gpt-4o-mini 128K | o3-mini 200K | gpt-4.1 128K", + "disposition": "outdated" + }, + { + "id": "ms-ai-security/cost-optimization/token-counting-optimization.md#16", + "judge_verdict": "not_grounded", + "rule": "R3", + "evidence_url": "https://learn.microsoft.com/microsoft-copilot-studio/requirements-messages-management", + "evidence_quote": "| Classic answer | 1 Copilot Credit | No charge | | Generative answer | 2 Copilot Credits | No charge | ... Text and generative AI tools (basic) per 10 response 0.1 Copilot Credit per 1K tokens", + "reason": "Copilot Studio bills Generative Answers at a flat per-event rate (2 Copilot Credits, formerly 2 messages), not token-based Azure OpenAI billing; per-1K-token rates apply only to the separate 'Text and generative AI tools' (AI Builder) meters, so the claim's billing frame is replaced by the current credits model.", + "file": "skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md", + "line": 406, + "claim": "Copilot Studio: token-basert billing for Generative Answers (Azure OpenAI), message-basert billing for standard topics; token counting via AI Builder credits", + "disposition": "outdated" + } + ] + } + ] +} diff --git a/scripts/kb-eval/data/r71-payloads.json b/scripts/kb-eval/data/r71-payloads.json new file mode 100644 index 0000000..8dca3e4 --- /dev/null +++ b/scripts/kb-eval/data/r71-payloads.json @@ -0,0 +1,227 @@ +[ + { + "file": "skills/ms-ai-engineering/references/agent-orchestration/foundry-workflows-visual-orchestration.md", + "claim_count": 16, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/agent-orchestration/foundry-workflows-visual-orchestration.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/agent-orchestration/foundry-workflows-visual-orchestration.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#1\",\n \"claim\": \"Foundry Workflows: Status Public Preview, annonsert på Microsoft Ignite november 2025\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/agents/concepts/workflow\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#2\",\n \"claim\": \"Workflows er ett av tre agenttyper i Foundry: Prompt-based | Workflow | Hosted (preview, containeriserte agenter)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/agents/concepts/workflow\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#3\",\n \"claim\": \"Node-typer i Foundry Workflows: Invoke agent | If/else | Go to | For each | Set variable | Parse value | Send message | Ask a question\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/agents/concepts/workflow\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#4\",\n \"claim\": \"Foundry tilbyr tre startmaler for workflows: Sequential | Human-in-the-loop | Group chat\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/agents/concepts/workflow\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#5\",\n \"claim\": \"Agent Framework HITL: RequestPort (C#) / ctx.request_info() (Python) for custom workflows; ToolApprovalRequestContent for agent orchestrations; workflow pauser og emitter RequestInfoEvent; checkpoints bevarer pending requests\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/agent-framework/workflows/human-in-the-loop\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#6\",\n \"claim\": \"Verktøy tilgjengelige i agent-noder i workflows (samme katalog som enkelt-agenter): Code Interpreter, Bing Search, Azure AI Search, Key Vault, MCP-servere\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/agents/concepts/workflow\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#7\",\n \"claim\": \"Foundry MCP Server er i preview og eksponerer Foundry selv som MCP-endepunkt\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#8\",\n \"claim\": \"Azure Logic Apps: 1400+ triggere (HTTP, Events, Schedule, Queues, SaaS) | 400+ enterprise-koblinger (SAP, Salesforce, AS2, EDI)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#9\",\n \"claim\": \"Foundry Workflows datasuverenitet: Azure-regioner, Norway East støttet\",\n \"claim_type\": \"region\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#10\",\n \"claim\": \"Agent Application-endepunkt kalles med api-version=2025-11-15-preview (protocols/openai/responses)\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/agents/how-to/agent-applications?view=foundry\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#11\",\n \"claim\": \"Klienter som kaller publisert Agent Application-endepunkt må ha rollen Foundry User (tidligere Azure AI User — rolle-ID og kjernepermisjoner uendret)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/agents/how-to/agent-applications?view=foundry\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#12\",\n \"claim\": \"Per februar 2026: A/B-testing av workflows begrenset — all trafikk rutes til én aktiv deployment; trafikkdeling mellom deployments er på roadmap\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/agents/how-to/agent-applications?view=foundry\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#13\",\n \"claim\": \"Foundry Workflows har immutable versjonssystem: hvert lagring oppretter ny versjon, eksisterende versjoner kan ikke endres; rollback via republisering; kodereferanse :\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/agents/concepts/development-lifecycle?view=foundry\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#14\",\n \"claim\": \"Agent Monitoring Dashboard-metrikker: Token usage | Latency | Run success rate | Error rate | Evaluation metrics\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/observability/how-to/how-to-monitor-agents-dashboard?view=foundry\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#15\",\n \"claim\": \"Foundry OpenTelemetry span-typer for multi-agent observability: execute_task | agent_to_agent_interaction | agent.state.management | agent_planning | execute_tool\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/foundry-workflows-visual-orchestration.md#16\",\n \"claim\": \"Declarative Workflows (Agent Framework) YAML action-typer: Variable, Control Flow, Output, Agent/Tool Invocation (InvokeAzureAgent, InvokeFunctionTool, InvokeMcpTool), HTTP, HITL (Question, RequestExternalInput), Conversation — C# og Python; Conversation-actions er C#-only\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/agent-framework/workflows/declarative\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/agent-orchestration/foundry-workflows-visual-orchestration.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/agent-orchestration/semantic-kernel-agents-implementation.md", + "claim_count": 8, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/agent-orchestration/semantic-kernel-agents-implementation.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/agent-orchestration/semantic-kernel-agents-implementation.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/semantic-kernel-agents-implementation.md#1\",\n \"claim\": \"Microsoft Agent Framework 1.0 nådde GA 3. april 2026; produksjonsklart open-source rammeverk (.NET + Python)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/agent-framework/overview/agent-framework-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/semantic-kernel-agents-implementation.md#2\",\n \"claim\": \"Semantic Kernel og AutoGen har konvergert inn i MAF og er nå i vedlikeholds-/migrasjonsmodus (migration guides finnes); ny utvikling bør bygge på MAF\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/agent-framework/overview/agent-framework-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/semantic-kernel-agents-implementation.md#3\",\n \"claim\": \"De fem orkestreringsmønstrene Sequential, Concurrent, Handoff, Group Chat og Magentic er stabile (GA) i MAF 1.0 med streaming, checkpointing, human-in-the-loop og pause/resume\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/semantic-kernel/frameworks/agent/agent-orchestration/\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/semantic-kernel-agents-implementation.md#4\",\n \"claim\": \"Semantic Kernel Agent Framework har fem hovedagent-typer: ChatCompletionAgent | OpenAIAssistantAgent | AzureAIAgent | OpenAIResponsesAgent | CopilotStudioAgent\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/semantic-kernel/frameworks/agent/agent-architecture\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/semantic-kernel-agents-implementation.md#5\",\n \"claim\": \"Nøkkelforskjell: Semantic Kernel bruker Kernel-objektet som sentral orkestreringsenhet; Microsoft Agent Framework bruker IChatClient (Microsoft.Extensions.AI) wrapped av ChatClientAgent\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/agent-framework/overview/agent-framework-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/semantic-kernel-agents-implementation.md#6\",\n \"claim\": \"MAF legger til enterprise-features: OpenTelemetry observability, Microsoft Entra-integrasjon, Agent-to-Agent (A2A) protocol, Model Context Protocol (MCP)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/agent-framework/overview/agent-framework-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/semantic-kernel-agents-implementation.md#7\",\n \"claim\": \"Stateful agents (AzureAIAgent, OpenAIAssistantAgent) lagrer state serverside og krever matching thread-type (f.eks. AzureAIAgent + AzureAIAgentThread); ChatCompletionAgent er stateless med lokal ChatHistory\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/semantic-kernel/frameworks/agent/agent-architecture\"\n },\n {\n \"id\": \"ms-ai-engineering/agent-orchestration/semantic-kernel-agents-implementation.md#8\",\n \"claim\": \"Semantic Kernel: MIT License | Microsoft Agent Framework: MIT License\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/agent-orchestration/semantic-kernel-agents-implementation.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/api-management/apim-ai-gateway-overview.md", + "claim_count": 7, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/api-management/apim-ai-gateway-overview.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/api-management/apim-ai-gateway-overview.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/api-management/apim-ai-gateway-overview.md#1\",\n \"claim\": \"APIM tier-støtte for AI gateway: Consumption = begrenset (ingen circuit breaker, semantic caching ja, ingen token limit by key); Developer/Basic v2/Standard v2 = full; Premium = full + multi-region\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/apim-ai-gateway-overview.md#2\",\n \"claim\": \"Token-policies i APIM: azure-openai-token-limit (Azure OpenAI) | llm-token-limit (alle LLM-er) | azure-openai-emit-token-metric | llm-emit-token-metric\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/apim-ai-gateway-overview.md#3\",\n \"claim\": \"APIM backend pool load balancing-moduser: Round-robin | Weighted | Priority-based | Session-aware (sesjonsaffinitet via cookie)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/apim-ai-gateway-overview.md#4\",\n \"claim\": \"APIM circuit breaker støtter dynamisk trip duration basert på Retry-After header fra backend-respons\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/apim-ai-gateway-overview.md#5\",\n \"claim\": \"Semantic caching i APIM krever Azure Managed Redis med RediSearch-modul\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/apim-ai-gateway-overview.md#6\",\n \"claim\": \"APIM AI-sikkerhetsfunksjoner: llm-content-safety policy (automatisk moderering via Azure AI Content Safety) | Credential Manager for OAuth-autorisasjon | sikring av MCP-servere via APIM\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/apim-ai-gateway-overview.md#7\",\n \"claim\": \"Bicep-ressurstype for APIM: Microsoft.ApiManagement/service@2023-09-01-preview (også backends; rolletildeling Microsoft.Authorization/roleAssignments@2022-04-01)\",\n \"claim_type\": \"version\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/api-management/apim-ai-gateway-overview.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/api-management/apim-authentication-oauth-managed-identity.md", + "claim_count": 6, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/api-management/apim-authentication-oauth-managed-identity.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/api-management/apim-authentication-oauth-managed-identity.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/api-management/apim-authentication-oauth-managed-identity.md#1\",\n \"claim\": \"Microsoft Entra ID (tidligere Azure AD) er primær identitetsleverandør for Azure-tjenester\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/api-management/apim-authentication-oauth-managed-identity.md#2\",\n \"claim\": \"RBAC-roller for Azure OpenAI: Cognitive Services OpenAI User (bruke deployments) | Cognitive Services OpenAI Contributor (opprette/administrere deployments) | Cognitive Services Contributor (full tilgang) | Reader (lese)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/api-management-authenticate-authorize-ai-apis\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/apim-authentication-oauth-managed-identity.md#3\",\n \"claim\": \"APIM-policy validate-azure-ad-token finnes med attributter tenant-id, header-name, failed-validation-httpcode og elementer client-application-ids, audiences, required-claims\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/validate-azure-ad-token-policy\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/apim-authentication-oauth-managed-identity.md#4\",\n \"claim\": \"APIM-policy authentication-managed-identity finnes med attributter resource, output-token-variable-name, ignore-error for tokenhenting via managed identity\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/authentication-managed-identity-policy\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/apim-authentication-oauth-managed-identity.md#5\",\n \"claim\": \"Ved import av API direkte fra Microsoft Foundry konfigurerer APIM automatisk managed identity-autentisering mot backend\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/azure-openai-api-from-specification\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/apim-authentication-oauth-managed-identity.md#6\",\n \"claim\": \"I APIM v2-tiers kan custom CA-sertifikater konfigureres direkte på backend-entiteten (thumbprint, subject name + issuer thumbprint, chain validation)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/backends\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/api-management/apim-authentication-oauth-managed-identity.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/api-management/backend-pool-management.md", + "claim_count": 10, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/api-management/backend-pool-management.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/api-management/backend-pool-management.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/api-management/backend-pool-management.md#1\",\n \"claim\": \"APIM backend pool-typer for AI-workloads: Round-robin | Weighted | Priority-based | Session-aware (sticky sessions)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/backend-pool-management.md#2\",\n \"claim\": \"Session-aware pools konfigureres med sessionAffinity (sessionId source: Cookie); session awareness bruker Set-Cookie header og klienten må håndtere cookies\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/backends\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/backend-pool-management.md#3\",\n \"claim\": \"APIM bruker circuit breaker-regler for å vurdere backend-helse i stedet for tradisjonelle health probes; ved trip rutes trafikk til neste prioritetsgruppe\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/backends\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/backend-pool-management.md#4\",\n \"claim\": \"Circuit breaker-egenskapen acceptRetryAfter: true gjør at breakeren respekterer Retry-After header fra backend (bl.a. Azure OpenAI 429)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/backends\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/backend-pool-management.md#5\",\n \"claim\": \"Azure OpenAI returnerer Retry-After header ved 429-responser; verdien kan være opptil 1 dag ved alvorlig overbelastning\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/api-management/backend-pool-management.md#6\",\n \"claim\": \"APIM innebygd status-endepunkt: GET https://{apim-name}-{region}-01.regional.azure-api.net/status-0123456789abcdef\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/api-management/backend-pool-management.md#7\",\n \"claim\": \"APIM-begrensning: maks 30 backends per pool\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/backends\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/backend-pool-management.md#8\",\n \"claim\": \"Bicep-ressurstype for APIM-backends: Microsoft.ApiManagement/service/backends@2023-09-01-preview\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/backends\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/backend-pool-management.md#9\",\n \"claim\": \"Circuit breaker-regler i APIM er tilnærmede (approximate) pga. distribuert arkitektur — ulike gateway-instanser synkroniserer ikke circuit state\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities\"\n },\n {\n \"id\": \"ms-ai-engineering/api-management/backend-pool-management.md#10\",\n \"claim\": \"Log Analytics-tabellen ApiManagementGatewayLlmLog finnes med kolonner TotalTokens, PromptTokens, CompletionTokens, BackendUrl (og ApiManagementGatewayLogs med BackendResponseCode)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/api-management/backend-pool-management.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-api-best-practices.md", + "claim_count": 12, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/ai-services-api-best-practices.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/ai-services-api-best-practices.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#1\",\n \"claim\": \"Azure OpenAI SDK-er (Python, .NET, Go) retrier automatisk 408|429|500|502|503|504 — opptil 3 ganger med eksponentiell backoff\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#2\",\n \"claim\": \"OpenAI Python SDK (Azure): api_version=2024-10-21|max_retries default: 2\",\n \"claim_type\": \"version\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#3\",\n \"claim\": \"Azure OpenAI Batch API-limits: max batch files uten utløp 500|med utløp 10 000|max input file size 200 MB (BYOS: 1 GB)|max requests per fil 100 000\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/batch\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#4\",\n \"claim\": \"Batch API: asynkrone batch-operasjoner med 50% lavere kostnad enn real-time API\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/batch\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#5\",\n \"claim\": \"Fail-fast-regioner for batching: Australia East|East US|Germany West Central|Italy North|North Central US|Poland Central|Sweden Central|Switzerland North|East US 2|West US\",\n \"claim_type\": \"region\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/batch\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#6\",\n \"claim\": \"Azure OpenAI Python SDK default timeout: 600s (10 min)\",\n \"claim_type\": \"version\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#7\",\n \"claim\": \"Azure Service Bus duplicate detection-vindu: default 10 minutter|min 20 sekunder|max 7 dager; basert på MessageId (evt. + PartitionKey)\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/service-bus-messaging/duplicate-detection\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#8\",\n \"claim\": \"Ved 429 på provisioned deployments returnerer tjenesten retry-after og retry-after-ms headers; standard SDK-oppførsel respekterer retry-after\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/provisioned-get-started\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#9\",\n \"claim\": \"Azure Monitor-metrikk for PTU-utnyttelse heter Provisioned-Managed Utilization V2\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/provisioned-get-started\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#10\",\n \"claim\": \"Provisioned reservation-rabatter: 1-årig commitment ~37%|3-årig commitment ~57%\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/provisioned-get-started\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#11\",\n \"claim\": \"SDK-er som støtter Microsoft Foundry: Python azure-ai-inference + openai (Azure-variant)|.NET Azure.AI.Inference + Azure.AI.OpenAI|JS/TS @azure/openai + @azure/ai-inference|Go github.com/openai/openai-go (med Azure endpoint)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-api-best-practices.md#12\",\n \"claim\": \"429-feil koster ingenting (ingen PTU-forbruk), men 400 content filter-feil, 408 timeout og finish_reason: content_filter koster\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/provisioned-get-started\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/ai-services-api-best-practices.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-networking-security.md", + "claim_count": 15, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/ai-services-networking-security.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/ai-services-networking-security.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#1\",\n \"claim\": \"Microsoft har omdøpt Azure Cognitive Services til Microsoft Foundry Tools (2026-04); API-endepunkter, RBAC-roller og ARM-ressurstyper beholder CognitiveServices i identifikatoren\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#2\",\n \"claim\": \"Private DNS-soner: privatelink.cognitiveservices.azure.com (AI Services)|privatelink.openai.azure.com (Azure OpenAI)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/cognitive-services-virtual-networks\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#3\",\n \"claim\": \"Target sub-resource for private endpoint (AI Services, Azure OpenAI): account\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/cognitive-services-virtual-networks\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#4\",\n \"claim\": \"Service tag for service endpoints: Microsoft.CognitiveServices\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/cognitive-services-virtual-networks\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#5\",\n \"claim\": \"Maks 100 VNet-regler per ressurs (kan kombineres med IP-regler)\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/cognitive-services-virtual-networks\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#6\",\n \"claim\": \"IP firewall-regler: kun offentlige IPv4-adresser; private RFC 1918-områder (10.*, 172.16-31.*, 192.168.*) ikke tillatt\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/cognitive-services-virtual-networks\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#7\",\n \"claim\": \"Azure AI Services støtter ikke direkte VNet-injeksjon (deployes ikke inn i kundens VNet); bruker private endpoints i stedet\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/cognitive-services-virtual-networks\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#8\",\n \"claim\": \"Nettverkssikkerhet (private endpoints, IP firewall, service endpoints) krever Basic tier (S0) eller høyere; Free tier ikke støttet\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/cognitive-services-virtual-networks\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#9\",\n \"claim\": \"RBAC-roller for tilgang: Cognitive Services User|Cognitive Services OpenAI User\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#10\",\n \"claim\": \"Trusted services bypass aktiveres med networkAcls.bypass: AzureServices; tillater Azure AI Search og Azure Machine Learning å kalle tjenesten via managed identity (JWT-validering)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/openai/how-to/on-your-data-configuration\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#11\",\n \"claim\": \"Custom DNS (on-premises) krever conditional forwarder til Azure DNS 168.63.129.16\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/cognitive-services-virtual-networks\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#12\",\n \"claim\": \"Service tags for portal-tilgang: AzureActiveDirectory|AzureFrontDoor.Frontend|AzureResourceManager|CognitiveServicesManagement|CognitiveServicesFrontEnd\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/cognitive-services-virtual-networks\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#13\",\n \"claim\": \"NSG-støtte ikke tilgjengelig for Azure AI Services; private link og disable public access støttet (per security baseline)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/security/benchmark/azure/baselines/cognitive-services-security-baseline\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#14\",\n \"claim\": \"Custom subdomain er påkrevd for private endpoints på AI Services\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/cognitive-services-virtual-networks\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-networking-security.md#15\",\n \"claim\": \"Power Automate/Power Apps mot private endpoints krever On-premises data gateway; Azure Relay hybrid connection ikke støttet direkte; custom connector må bruke public endpoint med IP firewall\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/ai-services-networking-security.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-vs-foundry-tools-selection.md", + "claim_count": 16, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/ai-services-vs-foundry-tools-selection.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/ai-services-vs-foundry-tools-selection.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#1\",\n \"claim\": \"Microsoft har endret terminologi Cognitive Services → Foundry Tools (2025); resource type kind: AIServices er konsistent\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#2\",\n \"claim\": \"Personalizer er utgått (retired)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/architecture/data-guide/technology-choices/ai-services\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#3\",\n \"claim\": \"Foundry resource type: kind AIServices med capabilities Agents|Evaluations|Azure OpenAI|Speech|Vision|Language|Content Understanding; legacy Azure OpenAI resource: kind OpenAI (kun OpenAI-modeller)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/concepts/resource-types\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#4\",\n \"claim\": \"Azure AI Hub (kind: Hub) er deprecated og migreres til Foundry resource type (Juni 2025+)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/concepts/resource-types\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#5\",\n \"claim\": \"Foundry Model Catalog: over 1 900 modeller fra Microsoft, OpenAI, Anthropic, Meta, Mistral, Cohere\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#6\",\n \"claim\": \"Azure OpenAI modellserie 2026-06: GPT-5-generasjonen (GPT-5/5.1/5.2/5.4/5.5 + -mini/-nano/-codex) er flaggskip for reasoning/chat; GPT-5.5 nyest\",\n \"claim_type\": \"version\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#7\",\n \"claim\": \"GPT-4.1, GPT-4.1-mini, GPT-4.1-nano: ~1M tokens kontekst; deployment-typer Standard|Global Standard|Provisioned\",\n \"claim_type\": \"version\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#8\",\n \"claim\": \"o4-mini, o3, o3-mini er eldre o-serie reasoning-modeller, erstattet av GPT-5-generasjonen for nye design; deployment Standard|Provisioned\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#9\",\n \"claim\": \"Global Standard deployment: globalt routet, ingen data residency, 9 % rimeligere, høy throughput\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#10\",\n \"claim\": \"Azure OpenAI er inkludert i Foundry resource type (kind: AIServices); legacy Azure OpenAI-ressurser (kind: OpenAI) kan oppgraderes til Foundry uten API-endringer\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/concepts/resource-types\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#11\",\n \"claim\": \"Content Understanding: GA v1.0 / 2025-11-01; multimodal, generative fields, reasoning\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-understanding/choosing-right-ai-tool\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#12\",\n \"claim\": \"QnA Maker er utgått (retired); erstattes av Language Understanding for FAQ-bots\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#13\",\n \"claim\": \"Norge-spesifikke Azure-regioner: Norway East (Oslo)|Norway West (Stavanger)\",\n \"claim_type\": \"region\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#14\",\n \"claim\": \"On-prem deployment: Foundry Tools ja (containers)|Microsoft Foundry nei (cloud-only)|Azure OpenAI nei (cloud-only)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#15\",\n \"claim\": \"Commitment Tiers (Foundry Tools): fast månedskostnad, spar 30-40 % vs pay-per-use\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/concepts/manage-costs\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-vs-foundry-tools-selection.md#16\",\n \"claim\": \"Agent-publisering til Agent 365 krever M365 E3 eller høyere (Azure subscription + M365 E3/E5)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/ai-services-vs-foundry-tools-selection.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/mlops-genaiops/prompt-flow-production-deployment.md", + "claim_count": 13, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/mlops-genaiops/prompt-flow-production-deployment.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/mlops-genaiops/prompt-flow-production-deployment.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/prompt-flow-production-deployment.md#1\",\n \"claim\": \"Prompt Flow (både Microsoft Foundry og Azure ML) pensjoneres 2027-04-20; anbefales ikke for ny utvikling; migrer til Microsoft Agent Framework; web authoring, VS Code-utvidelser og container images (promptflow-runtime, promptflow-runtime-stable, promptflow-python) får ikke lenger oppdateringer inkl. sikkerhetsoppdateringer\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/machine-learning/prompt-flow/migrate-prompt-flow-to-agent-framework?view=azureml-api-2\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/prompt-flow-production-deployment.md#2\",\n \"claim\": \"Default base image: mcr.microsoft.com/azureml/promptflow/promptflow-runtime-stable:latest\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/how-to-deploy-to-code?view=azureml-api-2\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/prompt-flow-production-deployment.md#3\",\n \"claim\": \"PROMPTFLOW_SERVING_ENGINE: flask (default)|fastapi krever SDK >= 1.10.0\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/how-to-deploy-to-code?view=azureml-api-2\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/prompt-flow-production-deployment.md#4\",\n \"claim\": \"PROMPTFLOW_WORKER_NUM default = CPU cores|PROMPTFLOW_WORKER_THREADS default = 1\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/how-to-deploy-to-code?view=azureml-api-2\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/prompt-flow-production-deployment.md#5\",\n \"claim\": \"Azure Managed Online Endpoint krever Microsoft.PolicyInsights resource provider registrert\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/how-to/flow-deploy?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/prompt-flow-production-deployment.md#6\",\n \"claim\": \"Deployment-identitet krever rollen Azure Machine Learning Workspace Connection Secrets Reader; connections med Entra ID auth (Azure OpenAI) krever Cognitive Services OpenAI User\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/how-to/flow-deploy?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/prompt-flow-production-deployment.md#7\",\n \"claim\": \"Deployment tar 15-20 minutter (endpoint creation, model registration, deployment creation)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/how-to/flow-deploy?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/prompt-flow-production-deployment.md#8\",\n \"claim\": \"App Insights-metrics under namespace 'promptflow standard metrics': token_consumption|flow_latency|flow_request|node_latency|node_request|rpc_latency|rpc_request|flow_streaming_response_duration\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/how-to/develop/trace-production-sdk?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/prompt-flow-production-deployment.md#9\",\n \"claim\": \"Prompt Flow serving eksponerer /feedback endpoint for post-inference feedback; traceparent-header korrelerer feedback til original request trace\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/how-to/develop/trace-production-sdk?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/prompt-flow-production-deployment.md#10\",\n \"claim\": \"Prompt Flow er open source (MIT license) + Azure-managed variant; ingen lisenskostnad for SDK/CLI\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/prompt-flow-production-deployment.md#11\",\n \"claim\": \"Reserved instances: opptil 72% rabatt for forutsigbar last\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/prompt-flow-production-deployment.md#12\",\n \"claim\": \"Concurrency-formel: max_concurrent_requests_per_instance = worker_num × worker_threads × multiplier (1.0 hvis request time > 200ms, 1.5-2.0 hvis <= 200ms)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/how-to-deploy-to-code?view=azureml-api-2\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/prompt-flow-production-deployment.md#13\",\n \"claim\": \"Instance count minimum 3 for high availability ved deployment\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/how-to/flow-deploy?view=foundry-classic\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/mlops-genaiops/prompt-flow-production-deployment.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/rag-architecture/citation-tracking.md", + "claim_count": 10, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/rag-architecture/citation-tracking.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/rag-architecture/citation-tracking.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/rag-architecture/citation-tracking.md#1\",\n \"claim\": \"Azure AI Search agentic retrieval: grounding-overflaten (references + activity-log) er GA via REST 2026-04-01; LLM query planning (ikke-minimal reasoning effort) + answer synthesis + multi-turn messages forblir preview (2026-05-01-preview); portal/Foundry preview\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/agentic-retrieval-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/citation-tracking.md#2\",\n \"claim\": \"Azure AI Search REST API-versjoner for agentic retrieval: 2026-04-01 (GA) og 2026-05-01-preview\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/agentic-retrieval-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/citation-tracking.md#3\",\n \"claim\": \"Classic RAG (Azure AI Search + Azure OpenAI, 'On Your Data' med citation annotations) er GA\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/retrieval-augmented-generation-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/citation-tracking.md#4\",\n \"claim\": \"Semantic Ranking-score (Azure AI Search): skala 0.0–4.0\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/retrieval-augmented-generation-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/citation-tracking.md#5\",\n \"claim\": \"Vector Similarity (Cosine) i Azure AI Search: skala 0.333–1.0\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/retrieval-augmented-generation-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/citation-tracking.md#6\",\n \"claim\": \"Groundedness Pro: threshold-basert scoring, leveres av Azure AI Content Safety (per evaluerings-kall)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/citation-tracking.md#7\",\n \"claim\": \"Semantic Answer Confidence (Azure AI Search): 70% terskel\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/retrieval-augmented-generation-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/citation-tracking.md#8\",\n \"claim\": \"Semantic Ranker prising: gratis månedlig request-kvote + standard pay-as-you-go etter kvoten\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/citation-tracking.md#9\",\n \"claim\": \"Agentic retrieval faktureres token-basert (retrieval-tokens, ikke per query): månedlig gratis-kvote, deretter pay-as-you-go; Azure OpenAI faktureres separat for query planning/answer synthesis\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/agentic-retrieval-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/citation-tracking.md#10\",\n \"claim\": \"Azure AI Content Safety tilbyr groundedness-deteksjon med korreksjonsfunksjon\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/rag-architecture/citation-tracking.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-context-windows.md", + "claim_count": 11, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/rag-architecture/rag-context-windows.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/rag-architecture/rag-context-windows.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-context-windows.md#1\",\n \"claim\": \"GPT-4 Turbo (gpt-4 turbo-2024-04-09): 128k tokens context window\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/quotas-limits\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-context-windows.md#2\",\n \"claim\": \"GPT-4.1 series: ~1M tokens context window (max 1 047 576; standard deployments 300k; provisioned managed/batch 128k)\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/quotas-limits\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-context-windows.md#3\",\n \"claim\": \"GPT-4o: 128k tokens context window | gpt-4o-mini: 128k tokens context window\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/quotas-limits\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-context-windows.md#4\",\n \"claim\": \"o1: 200k context window | o3-mini: 200k input / 100k output\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/quotas-limits\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-context-windows.md#5\",\n \"claim\": \"Azure OpenAI quota system is Quota Tiers: Free Tier (Tier 0) plus Tier 1-6 (Tier 6 highest), quota auto-increases with usage; replaces former Default/Enterprise two-tier system; TPM/RPM defined per region, per subscription, per model/deployment type\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/openai/quotas-limits\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-context-windows.md#6\",\n \"claim\": \"GlobalStandard TPM examples (Tier 1 to Tier 6): gpt-4.1 1M to 45M | gpt-4o-mini 2M to 225M | o1 3M to 48M | o3-mini 5M to 80M; max TPM (Tier 6) = 225M\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/openai/quotas-limits\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-context-windows.md#7\",\n \"claim\": \"Azure OpenAI Assistants API supports max_prompt_tokens and max_completion_tokens on Run creation; truncation_strategy options are auto and last_messages\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/openai/concepts/assistants#context-window-management\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-context-windows.md#8\",\n \"claim\": \"For Assistants API File Search tool: set max_prompt_tokens to minimum 20,000; for longer conversations 50,000 or remove limit\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/openai/concepts/assistants#context-window-management\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-context-windows.md#9\",\n \"claim\": \"Azure OpenAI Batch API gives 50% discount for non-interactive workloads\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/quotas-limits\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-context-windows.md#10\",\n \"claim\": \"Azure OpenAI Batch quota (per Quota Tier): gpt-4.1 ~500M tokens/month (upper tier), ~30M (lower tier); gpt-4o ~500M (upper), ~30M (lower)\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/quotas-limits\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-context-windows.md#11\",\n \"claim\": \"Prompt Flow (Microsoft Foundry) shows estimated token count per node in the flow\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/rag-architecture/rag-context-windows.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-governance/references/responsible-ai/content-safety-implementation.md", + "claim_count": 12, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-governance/references/responsible-ai/content-safety-implementation.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-governance/references/responsible-ai/content-safety-implementation.md`)\n\n[\n {\n \"id\": \"ms-ai-governance/responsible-ai/content-safety-implementation.md#1\",\n \"claim\": \"Content Safety feature statuses: Analyze Text GA | Analyze Image GA | Prompt Shields GA | Groundedness Detection Preview | Protected Material Text GA | Protected Material Code GA | Custom Categories (Standard) Preview | Custom Categories (Rapid) Preview | Blocklists GA\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-safety/overview\"\n },\n {\n \"id\": \"ms-ai-governance/responsible-ai/content-safety-implementation.md#2\",\n \"claim\": \"Content Safety input limits: Analyze Text max 10K chars | Analyze Image JPEG/PNG/GIF/BMP/TIFF/WEBP max 4MB | Prompt Shields text max 10K chars | Groundedness query+sources max 55K chars | Protected Material Text min 110 chars\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-safety/overview\"\n },\n {\n \"id\": \"ms-ai-governance/responsible-ai/content-safety-implementation.md#3\",\n \"claim\": \"Content analysis uses four harm categories (hate, sexual, violence, self-harm) with severity levels 0-6 (0 Safe, 2 Low, 4 Medium, 6 High); 0 is annotate-only, 2/4/6 configurable\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-safety/concepts/harm-categories\"\n },\n {\n \"id\": \"ms-ai-governance/responsible-ai/content-safety-implementation.md#4\",\n \"claim\": \"Azure OpenAI default content filter blocks Medium (4) and High (6) severity for all four categories\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/default-safety-policies\"\n },\n {\n \"id\": \"ms-ai-governance/responsible-ai/content-safety-implementation.md#5\",\n \"claim\": \"Content filter response behavior: prompt blocked gives HTTP 400 with error.code=content_filter | completion blocked (non-streaming) gives 200 with finish_reason=content_filter | streaming blocked: stream stops, last chunk carries finish_reason | filter error: content_filter_results.error populated\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/foundry-models/concepts/content-filter\"\n },\n {\n \"id\": \"ms-ai-governance/responsible-ai/content-safety-implementation.md#6\",\n \"claim\": \"Copilot Studio cannot (per feb 2026) configure severity levels per category - uses Azure OpenAI deployment settings; custom blocklists can be enabled in Agent Settings\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-governance/responsible-ai/content-safety-implementation.md#7\",\n \"claim\": \"AI Builder Text generation uses Azure OpenAI with content filtering enabled by default and no configuration options (default Medium+High block)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-governance/responsible-ai/content-safety-implementation.md#8\",\n \"claim\": \"Microsoft 365 Copilot has its own content filtering policies that are not customer-configurable (severity levels cannot be adjusted; Microsoft-managed)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-governance/responsible-ai/content-safety-implementation.md#9\",\n \"claim\": \"Azure AI Content Safety available in West Europe and Norway East (via Azure OpenAI); models run in EU (no data transfer to USA)\",\n \"claim_type\": \"region\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-safety/overview\"\n },\n {\n \"id\": \"ms-ai-governance/responsible-ai/content-safety-implementation.md#10\",\n \"claim\": \"Azure AI Content Safety has PII detection for completions (names, addresses; Norwegian national ID format not officially supported), configurable to block or mask PII in LLM output\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-governance/responsible-ai/content-safety-implementation.md#11\",\n \"claim\": \"Customer-managed keys (CMK/BYOK) supported for Content Safety blocklist data; blocklist data stored in same region as resource, encrypted at rest; no prompts/completions stored for training\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/content-safety/data-privacy\"\n },\n {\n \"id\": \"ms-ai-governance/responsible-ai/content-safety-implementation.md#12\",\n \"claim\": \"Content Safety tiers/rate limits: F0 (Free) 5 RPS | S0 (Standard) 1000 RP10S | Groundedness 50 RPS limit\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-safety/overview\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-governance/references/responsible-ai/content-safety-implementation.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-security/references/ai-security-engineering/ai-prompt-shield-network.md", + "claim_count": 12, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-security/references/ai-security-engineering/ai-prompt-shield-network.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-security/references/ai-security-engineering/ai-prompt-shield-network.md`)\n\n[\n {\n \"id\": \"ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#1\",\n \"claim\": \"Two separate products: Azure AI Content Safety Prompt Shields is GA; AI Gateway Prompt Shield via Global Secure Access is Preview\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/entra/global-secure-access/how-to-ai-prompt-shield\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#2\",\n \"claim\": \"Prompt Shields endpoint: POST {endpoint}/contentsafety/text:shieldPrompt?api-version=2024-09-01\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/content-safety/quickstart-jailbreak\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#3\",\n \"claim\": \"Prompt Shields input limits: userPrompt max 10 000 chars | documents array max 5 documents per request | each document max 10 000 chars\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/content-safety/concepts/jailbreak-detection\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#4\",\n \"claim\": \"AI Gateway Prompt Shield has pre-configured extractors for: Microsoft Copilot, OpenAI ChatGPT, Anthropic Claude, Meta Llama, xAI Grok, Mistral, Cohere, Inflection Pi, Alibaba Qwen, plus custom JSON-based LLMs (custom URL + JSON path)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/entra/global-secure-access/how-to-ai-prompt-shield\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#5\",\n \"claim\": \"AI Gateway Prompt Shield limitations: text prompts only (no files); JSON-based GenAI apps only (not URL-encoded, e.g. Gemini); max 10 000 chars per prompt (longer prompts truncated)\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/entra/global-secure-access/how-to-ai-prompt-shield\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#6\",\n \"claim\": \"AI Gateway Prompt Shield prerequisites: Microsoft Entra Internet Access license, Entra-joined or hybrid-joined devices, Global Secure Access Administrator and Conditional Access Administrator roles\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/entra/global-secure-access/how-to-ai-prompt-shield\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#7\",\n \"claim\": \"APIM AI gateway mediates OpenAI Chat Completions/Responses, Anthropic Messages API (supported in v2 tiers), Google Vertex AI, remote MCP servers and A2A Agent APIs; unified model API (preview) exposes multiple backends via one OpenAI-compatible endpoint; AI gateway can integrate directly with Microsoft Foundry (preview)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities#security-and-safety\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#8\",\n \"claim\": \"APIM llm-content-safety policy: shield-prompt attribute; enforce-on-completions (inbound, also checks LLM responses); window-size default 10 000 chars; window-overlap-size; threshold 0-7; output-type FourSeverityLevels (0,2,4,6) or EightSeverityLevels (0-7); blocked requests return 403; supported categories Hate/SelfHarm/Sexual/Violence; supports blocklists; usable in inbound and outbound\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/api-management/llm-content-safety-policy\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#9\",\n \"claim\": \"APIM llm-content-safety policy available in tiers: Developer, Basic, Basic v2, Standard, Standard v2, Premium, Premium v2\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/api-management/llm-content-safety-policy\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#10\",\n \"claim\": \"Groundedness Detection: supports QnA and summarization tasks; domain options MEDICAL or GENERIC; two modes Non-Reasoning and Reasoning; Groundedness Correction is preview (returns correctedText); only English text supported\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/content-safety/concepts/groundedness\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#11\",\n \"claim\": \"Groundedness endpoint: POST {endpoint}/contentsafety/text:detectGroundedness?api-version=2024-09-01\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/content-safety/concepts/groundedness\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/ai-prompt-shield-network.md#12\",\n \"claim\": \"AI Gateway Prompt Shield (Preview) is included in the Microsoft Entra Internet Access license (licensed per user/month)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/entra/global-secure-access/how-to-ai-prompt-shield\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-security/references/ai-security-engineering/ai-prompt-shield-network.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-security/references/ai-security-engineering/zero-trust-ai-services.md", + "claim_count": 11, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-security/references/ai-security-engineering/zero-trust-ai-services.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-security/references/ai-security-engineering/zero-trust-ai-services.md`)\n\n[\n {\n \"id\": \"ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#1\",\n \"claim\": \"RBAC roles: Cognitive Services OpenAI User (5e0bd9bd-7b93-4f28-af87-19fc36ad61bd, inference) | Cognitive Services OpenAI Contributor (a001fd3d-188f-4b5d-821b-7da978bf7442, inference + deployment) | Cognitive Services User (a97b65f3-24c7-4388-baec-2e87135dc908, data plane) | Cognitive Services Contributor (25fbc0a9-bd7c-42a3-aa1a-3b75d497ee68, full control incl. keys)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/translator/document-translation/how-to-guides/create-use-managed-identities\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#2\",\n \"claim\": \"Managed Identity tokens are cached up to 24 hours; group/role membership changes can take hours to propagate; App Roles propagate faster than groups\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/managed-identity-best-practice-recommendations\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#3\",\n \"claim\": \"Microsoft now explicitly recommends avoiding API keys for Azure AI Services in production; Microsoft Foundry resource (formerly 'Foundry Tools', kind=AIServices) uses the same DefaultAzureCredential pattern across all AI services\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/openai/how-to/managed-identity\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#4\",\n \"claim\": \"CAE token lifetime: up to 28 hours (vs standard access token 1 hour, Managed Identity refresh token 24 hours); critical events revoke in seconds\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/entra/identity/conditional-access/concept-continuous-access-evaluation\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#5\",\n \"claim\": \"From 2025 CAE is configured via Conditional Access policies (Session controls) instead of a separate toggle; options Disabled, Basic (default), Strict\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/entra/identity/conditional-access/concept-continuous-access-evaluation\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#6\",\n \"claim\": \"Strict Location Enforcement is Preview, activated per Conditional Access policy, blocks tokens used outside approved network locations in near-realtime, and now supports Azure Government clouds\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/entra/identity/conditional-access/concept-continuous-access-evaluation\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#7\",\n \"claim\": \"AKS Workload Identity replaces AAD Pod Identity for giving pods Managed Identity\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#8\",\n \"claim\": \"Microsoft Foundry Managed Network Isolation modes: Allow Internet Outbound (default) and Allow Only Approved Outbound; Foundry auto-creates private endpoints\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/machine-learning/how-to-network-isolation-planning\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#9\",\n \"claim\": \"Virtual Network Integration for Dataverse is in Private Preview (Q1 2026)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/power-platform/admin/vnet-data-gateway\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#10\",\n \"claim\": \"Zero Trust Guidance Center is organized by technology pillars and business scenarios and now includes a dedicated 'Protect AI and data' scenario (security-adoption-scenario-secure-ai); Secure Future Initiative (SFI) has its own 'SFI patterns and best practices' section in the Zero Trust hub\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/security/zero-trust/\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/zero-trust-ai-services.md#11\",\n \"claim\": \"Azure AI security baseline (azure-openai-security-baseline) reflects Microsoft Cloud Security Benchmark (MCSB) v1.0\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/security/benchmark/azure/baselines/azure-openai-security-baseline\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-security/references/ai-security-engineering/zero-trust-ai-services.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-security/references/cost-optimization/azure-ai-foundry-cost-governance.md", + "claim_count": 9, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-security/references/cost-optimization/azure-ai-foundry-cost-governance.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-security/references/cost-optimization/azure-ai-foundry-cost-governance.md`)\n\n[\n {\n \"id\": \"ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#1\",\n \"claim\": \"Model Quota (TPM) scope: per subscription, per region, per model; default varies per Quota Tier (Free Tier (Tier 0) + Tier 1-6); ca. 100K-225M TPM; adjustable via quota request\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/quotas-limits\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#2\",\n \"claim\": \"VM Family Quota default 24-300 cores per subscription per region (depends on subscription type), adjustable via support request\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/how-to/hub-quota\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#3\",\n \"claim\": \"Compute Instances: 500 total compute limit per region, adjustable up to 2500 via quota UI, beyond via support\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/how-to/hub-quota\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#4\",\n \"claim\": \"Serverless API Quota: 200K TPM and 1K RPM per deployment; one deployment per model per project by default\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/quotas-limits\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#5\",\n \"claim\": \"Azure OpenAI has no hard-limit budget enforcement (unlike OpenAI API); Azure budgets only send alerts and do not stop consumption automatically\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/concepts/manage-costs\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#6\",\n \"claim\": \"Dynamic quota is Preview; lets deployments opportunistically use unused capacity beyond baseline quota; has no upper cap for spending control\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/openai/how-to/dynamic-quota\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#7\",\n \"claim\": \"The underlying Azure resource (API kind AIServices) is renamed Microsoft Foundry resource (successor/renaming of 'Foundry Tools'); projects function as folders grouping work under one resource\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#8\",\n \"claim\": \"Foundry RBAC roles renamed (role IDs and core permissions unchanged): Azure AI User to Foundry User | Azure AI Owner to Foundry Owner | Azure AI Account Owner to Foundry Account Owner | Azure AI Project Manager to Foundry Project Manager\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/azure-ai-foundry-cost-governance.md#9\",\n \"claim\": \"Azure Owner and Contributor give only management access (create/configure resources, quota/cost) - not development data actions; Foundry User (least privilege for developers) or Foundry Owner required to build\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-security/references/cost-optimization/azure-ai-foundry-cost-governance.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-security/references/cost-optimization/batch-processing-cost-reduction.md", + "claim_count": 11, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-security/references/cost-optimization/batch-processing-cost-reduction.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-security/references/cost-optimization/batch-processing-cost-reduction.md`)\n\n[\n {\n \"id\": \"ms-ai-security/cost-optimization/batch-processing-cost-reduction.md#1\",\n \"claim\": \"Azure OpenAI Batch API gir 50% kostnadsreduksjon vs standard global deployment og bruker separat enqueued token-kvote\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/openai/how-to/batch\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/batch-processing-cost-reduction.md#2\",\n \"claim\": \"To batch deployment-typer: Global-Batch (globalt distribuert) | Data Zone Batch (regionsbasert)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/deployment-types\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/batch-processing-cost-reduction.md#3\",\n \"claim\": \"Exponential backoff (automatisk retry av store batch-jobber) støttes i utvalgte regioner\",\n \"claim_type\": \"region\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/openai/how-to/batch\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/batch-processing-cost-reduction.md#4\",\n \"claim\": \"Batch-jobber har 24-timers target completion window; jobber som tar lengre tid expires ikke; kansellering når som helst, betaler kun for fullført arbeid\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/openai/how-to/batch\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/batch-processing-cost-reduction.md#5\",\n \"claim\": \"Batch-støttede modeller (juni 2026): gpt-5.1 2025-11-13 | gpt-5 2025-08-07 | o3 2025-04-16 | o4-mini 2025-04-16 | gpt-4.1 2025-04-14 | gpt-4.1-mini 2025-04-14 | gpt-4.1-nano 2025-04-14 | o3-mini 2025-01-31 | gpt-4o 2024-05-13/2024-08-06/2024-11-20 | gpt-4o-mini 2024-07-18\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/openai/how-to/batch\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/batch-processing-cost-reduction.md#6\",\n \"claim\": \"Tilgang til gpt-5 og o3 i batch krever registrering\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/openai/how-to/batch\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/batch-processing-cost-reduction.md#7\",\n \"claim\": \"gpt-5.1, gpt-5 og o3-mini er ikke tilgjengelige i alle Data Zone Batch-regioner\",\n \"claim_type\": \"region\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/openai/how-to/batch\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/batch-processing-cost-reduction.md#8\",\n \"claim\": \"o3-mini er ikke støttet med API-versjon 2024-10-21; nyere modeller med Global Batch krever v1 API\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/openai/how-to/batch\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/batch-processing-cost-reduction.md#9\",\n \"claim\": \"Batch støttes ikke med Assistants API eller Azure OpenAI On Your Data\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/openai/how-to/batch\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/batch-processing-cost-reduction.md#10\",\n \"claim\": \"Batch input file (JSONL, purpose \\\"batch\\\") kan settes med expiration 14-30 dager fra upload\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/batch?pivots=programming-language-python#upload-batch-file\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/batch-processing-cost-reduction.md#11\",\n \"claim\": \"Azure OpenAI Batch API har status GA\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/openai/whats-new#august-2024\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-security/references/cost-optimization/batch-processing-cost-reduction.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md", + "claim_count": 16, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md`)\n\n[\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#1\",\n \"claim\": \"o200k_base er default tiktoken-encoding for gpt-4o, o1, o3\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://github.com/openai/tiktoken\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#2\",\n \"claim\": \"Azure OpenAI On Your Data er deprecated og pensjoneres 2026-10-14; migrer til Foundry Agent Service + Foundry IQ\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/openai/concepts/use-your-data#token-usage-estimation-for-azure-openai-on-your-data\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#3\",\n \"claim\": \"On Your Data RAG token-fordeling: 20% av context window reservert for model response, 80% deles mellom meta prompt/spørsmål/historikk/chunks; user question og history capped ved 2000 tokens\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/openai/concepts/use-your-data#token-usage-estimation-for-azure-openai-on-your-data\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#4\",\n \"claim\": \"Prompt caching automatisk aktivert for GPT-4o og nyere; minimum prompt length 1024 tokens for cache hit; granularitet 128 tokens etter første 1024; in-memory cache TTL 5-10 min idle, max 1 time\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/prompt-caching\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#5\",\n \"claim\": \"Extended cache TTL opptil 24 timer via prompt_cache_retention:\\\"24h\\\" for GPT-5-serien + gpt-4.1; default 24h for nyere modeller\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/prompt-caching\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#6\",\n \"claim\": \"Cached tokens: 50% rabatt på Standard deployment; opptil 100% rabatt på Provisioned (inkludert i PTU-pris)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/prompt-caching\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#7\",\n \"claim\": \"Samme prefix + prompt_cache_key over ~15 req/min gjør at noen requests overflower til ekstra maskiner og cache-effektiviteten faller\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/prompt-caching\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#8\",\n \"claim\": \"Token-limit per chunk for embeddings: text-embedding-ada-002 8191 tokens | text-embedding-3-small/large 8191 tokens\",\n \"claim_type\": \"version\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#9\",\n \"claim\": \"gpt-4o-mini fine-tuning: training example max 64536 tokens, input limit 128000 tokens\",\n \"claim_type\": \"version\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#10\",\n \"claim\": \"On Your Data gjennomsnittlig token-bruk per request: intent prompt ~1366 | generation prompt ~4297 | response ~111 | intent output ~25 | totalt ~5800 tokens\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/openai/concepts/use-your-data#token-usage-estimation-for-azure-openai-on-your-data\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#11\",\n \"claim\": \"RFT (Reinforcement Fine-Tuning) faktureres tidsbasert (ikke token-basert) med per-job-tak $5000; ved tak pauses training og deploybart checkpoint opprettes; ingen kostnad for kø/failed jobs/kansellering før training\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/fine-tuning-cost-management\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#12\",\n \"claim\": \"Fine-tuned model hosting: $1.70/time per deployment (Standard/Global Standard), påløper selv uten bruk\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/fine-tuning-cost-management\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#13\",\n \"claim\": \"Developer tier fine-tuning: ingen hosting fee, men deployment auto-deletes etter 24 timer\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/fine-tuning-cost-management\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#14\",\n \"claim\": \"Context windows: gpt-5 400K | gpt-5-mini 400K | gpt-4o 128K | gpt-4o-mini 128K | o3-mini 200K | gpt-4.1 128K\",\n \"claim_type\": \"version\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#15\",\n \"claim\": \"Image token conversion: low-res (<512x512) 85 tokens flat; high-res resizes til 2048x2048, splittes i 512x512 tiles, 170 tokens per tile + 85 base\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/token-counting-optimization.md#16\",\n \"claim\": \"Copilot Studio: token-basert billing for Generative Answers (Azure OpenAI), message-basert billing for standard topics; token counting via AI Builder credits\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-security/references/cost-optimization/token-counting-optimization.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md", + "claim_count": 17, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md`)\n\n[\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#1\",\n \"claim\": \"gpt-4.1 kontekstvindu: 1047576 tokens (full), 128000 tokens (standard og provisioned deployments), 300000 tokens (batch deployments)\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#2\",\n \"claim\": \"gpt-4.1: treningsdata til og med mai 2024; versjon 2025-04-14\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#3\",\n \"claim\": \"GPT-4.1-serien deployment-typer: Global Standard, Data Zone Standard, Regional (Standard og Provisioned)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#4\",\n \"claim\": \"Priority Processing er tilgjengelig for gpt-4.1, ikke for gpt-4.1-mini/nano\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#5\",\n \"claim\": \"gpt-5 og gpt-5-codex krever registrering og godkjenning; gpt-5-mini, gpt-5-nano, gpt-5-chat har ingen registreringskrav\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#6\",\n \"claim\": \"GPT-5 kontekstvindu: 400000 tokens (272K input / 128K output for resonneringsmodeller)\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#7\",\n \"claim\": \"PTU-gjennomstrømming (input TPM per PTU / min PTU global / min PTU regional): gpt-5 4750/15/50 | gpt-5-mini 23750/15/25 | gpt-4.1 3000/15/50 | gpt-4.1-mini 14900/15/25 | gpt-4.1-nano 59400/15/25 | o4-mini 5400/15/25; latens-SLA p50: 99% > 50/80/80/90/100/90 TPS\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/openai/concepts/provisioned-throughput-billing?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#8\",\n \"claim\": \"PTU input/output-ratio: gpt-4.1 1 output token = 4 input tokens; gpt-5 1 output token = 8 input tokens\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/openai/concepts/provisioned-throughput-billing?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#9\",\n \"claim\": \"Copilot Credits-takstnivåer: gpt-4.1-mini Basic | gpt-4.1 Standard | gpt-5-chat (preview) Standard | gpt-5-reasoning (preview) Premium | o3 Premium | Claude Sonnet 4.5 (experimental) Standard | Claude Opus 4.5 (experimental) Premium\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/microsoft-copilot-studio/prompt-model-settings\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#10\",\n \"claim\": \"gpt-4.1-mini er standardmodellen i Copilot Studio (erstattet GPT-4o mini); GPT-4o og GPT-4o mini er kun tilgjengelig i US government-regioner\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/microsoft-copilot-studio/prompt-model-settings\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#11\",\n \"claim\": \"Claude Sonnet 4.5 og Claude Opus 4.5 er tilgjengelig i Copilot Studio (experimental, 200K kontekstvindu)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/microsoft-copilot-studio/prompt-model-settings\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#12\",\n \"claim\": \"Parallelle verktøykall støttes IKKE ved Minimal reasoning_effort på GPT-5; bruk Low/Medium/High for agentbruk\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/foundry-models/how-to/model-choice-guide?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#13\",\n \"claim\": \"Microsoft Foundry Model Router er GA og gir opptil 60% besparelse vs å alltid bruke GPT-5-familien (dokumentert av Microsoft)\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#14\",\n \"claim\": \"Norway East: kun gpt-4o/gpt-4o-mini tilgjengelig som Norge-resident (Standard/Regional PTU); gpt-4.1, o-serien (o3/o4-mini/o3-mini/o1) og GPT-5-familien finnes ikke som Regional/PTU i Norway East; eneste GPT-5 med EU-residens i Norway East er gpt-5.5 via Data Zone Standard\",\n \"claim_type\": \"region\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#15\",\n \"claim\": \"Copilot Studio prompt-builder modell-lineup: gpt-4.1-mini (default, Basic, 128K, trent t.o.m. juni 2024) | gpt-4.1 (Standard, 128K, juni 2024) | gpt-5-chat (Standard, 128K, sept 2024) | gpt-5-reasoning (Premium, 400K, sept 2024) | gpt-5.2-reasoning (Premium, 400K, okt 2024) | gpt-5.3-chat (Standard, managed) | Claude Sonnet 4.6 (Standard, 200K) | Claude Opus 4.6 (Premium, 200K) | Grok 4.1 Fast (Standard, eksperimentell/ikke anbefalt)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/microsoft-copilot-studio/prompt-model-settings\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#16\",\n \"claim\": \"GPT-5-tilgjengelighet: gpt-5 GA (begrenset, krever godkjenning aka.ms/oai/gpt5access) | gpt-5-mini GA | gpt-5-nano GA | gpt-5-chat Preview (2 versjoner) | gpt-5-codex GA (begrenset) | gpt-5-pro GA (begrenset, kun MCA-E/Default-abonnementer)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure?view=foundry-classic\"\n },\n {\n \"id\": \"ms-ai-security/cost-optimization/gpt5-gpt41-pricing-models.md#17\",\n \"claim\": \"AI Builder: prompt builder credits inkludert i premium Power Platform-planer (500 credits/bruker/mnd); default modell gpt-4.1-mini (Basic rate)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-security/references/cost-optimization/gpt5-gpt41-pricing-models.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-security/references/ai-security-engineering/entra-agent-id-zero-trust.md", + "claim_count": 13, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-security/references/ai-security-engineering/entra-agent-id-zero-trust.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-security/references/ai-security-engineering/entra-agent-id-zero-trust.md`)\n\n[\n {\n \"id\": \"ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#1\",\n \"claim\": \"Microsoft Entra Agent ID har status Public Preview (utvidet etter Ignite 2025)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/entra/id-governance/agent-id-governance-overview\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#2\",\n \"claim\": \"Tre nye Agent ID RBAC-roller: Agent ID Administrator | Agent ID Developer | Agent Registry Administrator\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/entra/fundamentals/whats-new-ignite-2025\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#3\",\n \"claim\": \"Copilot Studio (preview, aktiveres per miljø i Power Platform Admin Center) oppretter automatisk Entra Agent ID for hver ny agent, knyttet til Microsoft Copilot Studio agent identity blueprint med Blueprint ID 25664c89-cea5-4ab6-b924-a54fd8a19ae0\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/microsoft-copilot-studio/admin-use-entra-agent-identities\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#4\",\n \"claim\": \"En agentidentitet er en spesialisert service principal med object ID og app ID alltid lik; ingen passord eller credentials — autentiseres kun via access tokens utstedt til plattformen agenten kjører på\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/entra/agent-id/identity-platform/what-is-agent-id\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#5\",\n \"claim\": \"Conditional Access for agenter krever Microsoft Entra ID P1 eller P2 og Microsoft Agent 365-lisens per bruker (lisenshåndhevelse kommer snart); nettverkskontroller for agenter krever Microsoft Entra Internet Access\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/entra/identity/conditional-access/agent-id\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#6\",\n \"claim\": \"Agent risk (high/medium/low) fra Identity Protection er tilgjengelig som CA-condition; «Agent execution environments»-condition er i Preview for managed endpoints (Windows 365 Cloud PCs for Agents)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/entra/identity/conditional-access/agent-id\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#7\",\n \"claim\": \"CA gjelder IKKE for: blueprint sine Graph-kall (oppretting av agentidentitet/agent user); intermediary token exchange på AAD Token Exchange Endpoint: Public (Resource ID fb60f99c-7a34-4190-8149-302f77469936); når Security defaults er aktivert; eller når agent bruker API-nøkkel\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/entra/identity/conditional-access/agent-id\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#8\",\n \"claim\": \"Blokkerte rettigheter for agentidentiteter: rollene Global Administrator, Privileged Role Administrator, User Administrator; Graph-tillatelsene Application.ReadWrite.All, RoleManagement.ReadWrite.All, User.ReadWrite.All, Directory.AccessAsUser.All\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/entra/agent-id/identity-professional/authorization-agent-id\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#9\",\n \"claim\": \"Foundry: første agent i et prosjekt oppretter automatisk standard agent identity blueprint + standard agentidentitet; upubliserte agenter deler prosjektidentitet; publisering oppretter dedikert blueprint + unik identitet og RBAC må tildeles på nytt\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/agents/concepts/agent-identity?view=foundry\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#10\",\n \"claim\": \"Opt-out fra Entra Agent Identity per miljø i Copilot Studio er midlertidig — Microsoft vil gjøre det obligatorisk for alle nye agenter i fremtiden\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/microsoft-copilot-studio/admin-use-entra-agent-identities\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#11\",\n \"claim\": \"Statustabell: Entra Agent ID kjerne/Agent Registry/Identity Protection for agenter/Global Secure Access for agenter = Public Preview (Frontier-program); Foundry-integrasjon Public Preview (alle Foundry-brukere); Conditional Access for agenter Public Preview; AI Prompt Shield nytt via Entra Internet Access; App Service/Azure Functions agent identity og Teams Developer Portal agent blueprints nye\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/entra/fundamentals/whats-new-ignite-2025\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#12\",\n \"claim\": \"Full Entra Agent ID-funksjonalitet krever deltakelse i Microsoft Frontier-programmet og M365 Copilot-lisens (aktiveres via M365 admin center → Copilot → Settings → User access → Copilot Frontier); Foundry-integrert agentidentitet er tilgjengelig for alle Foundry-brukere uten Frontier\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/microsoft-agent-365/admin/capabilities-entra\"\n },\n {\n \"id\": \"ms-ai-security/ai-security-engineering/entra-agent-id-zero-trust.md#13\",\n \"claim\": \"Entra Agent ID er en del av Microsoft Agent 365, Microsofts kontrollplan for agenter på tvers av virksomheten\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/microsoft-agent-365/admin/capabilities-entra\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-security/references/ai-security-engineering/entra-agent-id-zero-trust.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-cost-optimization.md", + "claim_count": 10, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/ai-services-cost-optimization.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/ai-services-cost-optimization.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#1\",\n \"claim\": \"Commitment Tier-støttede tjenester: Speech to Text (Standard) | Text to Speech (Neural) | Text Translation (Standard) | Language Understanding (Text Requests) | Azure Language (Sentiment/Key Phrase/Language Detection/NER) | Vision OCR | Document Intelligence (Custom/Invoice)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/commitment-tier\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#2\",\n \"claim\": \"Commitment Tier forpliktelsesperiode: kalendermåned (web/connected containers) eller kalenderår (disconnected containers)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/commitment-tier\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#3\",\n \"claim\": \"Commitment Tier: auto-renewal valgfritt og kan endres frem til midnatt UTC siste dag i måneden; ikke refunderbar etter kjøp; overage faktureres til samme sats som tier\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/commitment-tier\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#4\",\n \"claim\": \"Commitment Tier kan IKKE brukes med multi-service Cognitive Services-ressurs; krever dedikert single-service ressurs\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/commitment-tier\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#5\",\n \"claim\": \"PTU deployment-typer: Regional Provisioned (data i én region) | Data Zone Provisioned (data innenfor data zone, f.eks. EU/US) | Global Provisioned (global lastbalansering)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/provisioned-throughput\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#6\",\n \"claim\": \"PTU-reservasjoner: 1-måneds eller 1-års Azure Reservations; ingen 3-årstermin for Foundry PTU; rabatt varierer per modellfamilie og term\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/provisioned-throughput-billing\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#7\",\n \"claim\": \"For GPT-4o og nyere modeller er TPM per PTU satt separat for input- og output-tokens\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/provisioned-throughput\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#8\",\n \"claim\": \"Azure OpenAI har ingen hard limit-funksjonalitet (i motsetning til OpenAI); automatisering via Action Groups krever custom utvikling\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/concepts/manage-costs\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#9\",\n \"claim\": \"API Management AI gateway styrer også Anthropic Messages API (v2-tiers) og Google Vertex AI, tilbyr unified model API (preview), og kan integreres direkte i Microsoft Foundry (preview) for sentral token-/kostnadsgovernance\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-cost-optimization.md#10\",\n \"claim\": \"PTU er kun for Azure OpenAI — ikke for øvrige Azure AI Services\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/provisioned-throughput\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/ai-services-cost-optimization.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md", + "claim_count": 16, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#1\",\n \"claim\": \"Azure AI Services (Foundry Models) deployment-typer: Global Standard (automatisk routing til regioner med kapasitet) | Data Zone Standard (processing innenfor geografisk sone US/EU) | Regional Standard (én spesifikk region) | Provisioned PTU (dedikert kapasitet, SLA på latency)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/deployment-types\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#2\",\n \"claim\": \"Azure API Management backend pool: inntil 30 backends per pool\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#3\",\n \"claim\": \"APIM AI gateway medierer flere LLM-skjemaer: OpenAI Chat Completions/Responses, Anthropic Messages API (API Management v2-tiers), Google Vertex AI, samt Foundry-modeller og ikke-Microsoft-leverandører som Amazon Bedrock; styrer også MCP-servere og A2A-agent-APIer som backends\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#4\",\n \"claim\": \"APIM Unified model API (ett OpenAI-kompatibelt endepunkt med automatisk format-oversettelse over flere backends) er i preview\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#5\",\n \"claim\": \"AI gateway i Microsoft Foundry (governance av modeller, agenter og verktøy fra Foundry-miljøet: token-kvoter, throttling, content safety, telemetri) er i preview\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#6\",\n \"claim\": \"APIM semantisk caching bruker Azure Managed Redis (RediSearch-kompatibel) med llm-semantic-cache-store/llm-semantic-cache-lookup-policyer\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#7\",\n \"claim\": \"Azure OpenAI 429-responsens Retry-After-header kan være opptil 24 timer\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/backends\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#8\",\n \"claim\": \"Azure AI Search zone redundancy krever Standard tier eller høyere + minimum 3 replicas; ingen built-in disaster recovery (krever manuell gjenoppbygging eller support-kontakt)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/reliability/reliability-ai-search\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#9\",\n \"claim\": \"Azure AI Search har ingen native multi-region replication — separat service per region, geo-replication må implementeres selv\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/reliability/reliability-ai-search\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#10\",\n \"claim\": \"Cosmos DB continuous backup gir 7-day PITR (point-in-time restore) for Foundry Agent Service disaster recovery\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/architecture/ai-ml/architecture/baseline-microsoft-foundry-chat\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#11\",\n \"claim\": \"Ingen Azure OpenAI-region i Norge per 2026-02; nærmeste regioner er Sweden Central og West Europe\",\n \"claim_type\": \"region\",\n \"evidence_url\": \"https://azure.microsoft.com/global-infrastructure/services/\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#12\",\n \"claim\": \"APIM-tiers: Developer (1M calls, ingen SLA) | Basic (1M calls, SLA, 1 unit max) | Standard (10M calls, multi-region, 4 units) | Premium (unlimited, multi-region, VNet, 10+ units)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#13\",\n \"claim\": \"APIM circuit breaker er ikke tilgjengelig i Consumption tier; Standard tier minimum for AI gateway\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/api-management/backends\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#14\",\n \"claim\": \"Caching (prompt caching) for Azure OpenAI omtales som planlagt feature (ikke tilgjengelig ennå)\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#15\",\n \"claim\": \"Multi-Backend Gateway-guiden bekrefter fire gateway-topologier: (1) multiple model deployments i single instance, (2) multiple instances same region/single subscription, (3) multiple instances same region/multiple subscriptions, (4) multiple instances multi-region; tagger 'Foundry Tools' og 'Azure OpenAI in Foundry Models'\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/architecture/ai-ml/guide/azure-openai-gateway-multi-backend\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-enterprise-architecture.md#16\",\n \"claim\": \"Azure OpenAI er del av Foundry Models (tidligere 'OpenAI in Azure'), nå konsolidert under Foundry Tools; Azure AI Services het tidligere Cognitive Services\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/deployment-types\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/ai-services-enterprise-architecture.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-governance-compliance.md", + "claim_count": 12, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/ai-services-governance-compliance.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/ai-services-governance-compliance.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#1\",\n \"claim\": \"Purview Data Security Posture Management (DSPM) for AI støttes for Azure AI Services via Foundry; Enterprise AI apps-dekningen omfatter Anthropic Claude (Enterprise) i tillegg til Copilot\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/purview/ai-azure-services\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#2\",\n \"claim\": \"Purview-kapabiliteter støttet for AI Services: Compliance Manager | DSPM for AI | Data Classification | Sensitivity Labels | DLP (via Entra-registrerte apps) | Audit Logging (Unified Audit Log, fanger prompts/responses) | Data Lifecycle Management | eDiscovery\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/purview/ai-azure-services\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#3\",\n \"claim\": \"Purview for Azure AI Services (Foundry) aktiveres via Microsoft Foundry Portal ('Microsoft Purview Data Security' i Control Plane) eller via Microsoft Defender for Cloud ('Data Security for Azure AI with Microsoft Purview')\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/purview/ai-azure-services\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#4\",\n \"claim\": \"Purview-aktivering for AI Services krever pay-as-you-go billing i Purview; audit logging er inkludert i Purview-lisensen\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/purview/ai-azure-services\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#5\",\n \"claim\": \"AI Services diagnostikklogg-kategorier: Audit (API-kall, autentisering) | RequestResponse (full forespørsel/respons inkl. prompts) | AllMetrics; destinasjoner: Azure Storage, Log Analytics Workspace, Event Hub\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/diagnostic-logging\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#6\",\n \"claim\": \"Telemetry logs (objektnavn som indexer, skillsets) lagres globalt i 1,5 år for Microsoft-support (unntak fra Geography-residency)\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#7\",\n \"claim\": \"Diagnostic logs standard retention: 90 dager (Log Analytics default), justerbar 30-730 dager\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#8\",\n \"claim\": \"Azure Policy-definisjon 67121cc7-ff39-4ab8-b7e3-95b84dab487d = 'Azure AI services should enable data encryption with CMK'\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#9\",\n \"claim\": \"Keys kan lagres i HSM-backed Key Vault for FIPS 140-2 Level 3\",\n \"claim_type\": \"version\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#10\",\n \"claim\": \"Foundry støtter Microsoft Entra Agent Identity for unik identifisering av AI-agenter\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/cloud-adoption-framework/ai-agents/governance-security-across-organization\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#11\",\n \"claim\": \"Agent 365 gir sentralisert visning av alle AI-agenter i organisasjonen\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/cloud-adoption-framework/ai-agents/governance-security-across-organization\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-governance-compliance.md#12\",\n \"claim\": \"Purview-lisensiering: DLP Policies krever E5/F5-lisens (eller pay-as-you-go) | Sensitivity Labels krever E3/E5 | DSPM for AI ikke i standard lisens (pay-as-you-go tilgjengelig) | Audit (Unified Log) inkludert\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/ai-services-governance-compliance.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/ai-services-monitoring-logging.md", + "claim_count": 10, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/ai-services-monitoring-logging.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/ai-services-monitoring-logging.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-monitoring-logging.md#1\",\n \"claim\": \"Diagnostic settings log-kategorien Trace gjelder kun for Custom Question Answering\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/diagnostic-logging\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-monitoring-logging.md#2\",\n \"claim\": \"Azure AI Services rapporterer med ResourceProvider == 'MICROSOFT.COGNITIVESERVICES' i AzureDiagnostics-tabellen\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/openai/how-to/monitor-openai\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-monitoring-logging.md#3\",\n \"claim\": \"Log Analytics data retention: 90 dager gratis, deretter betalt\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-monitoring-logging.md#4\",\n \"claim\": \"Application Insights er OpenTelemetry-kompatibel APM med AI agent monitoring for Microsoft Foundry og Copilot Studio\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-monitoring-logging.md#5\",\n \"claim\": \"Application Insights har en Agents View spesialisert for AI agents (token usage, cost per session)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-monitoring-logging.md#6\",\n \"claim\": \"Out-of-box dashboards i Microsoft Foundry for Azure OpenAI: HTTP Requests | Tokens-Based Usage | PTU Utilization | Fine-tuning\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/openai/how-to/monitor-openai\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-monitoring-logging.md#7\",\n \"claim\": \"Azure OpenAI metric-navn: TokenTransaction | GeneratedTokens | ProcessedPromptTokens | ActiveTokens (PTU) | Requests | Http429\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/openai/how-to/monitor-openai\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-monitoring-logging.md#8\",\n \"claim\": \"Application Insights: 5 GB/måned gratis data ingestion per subscription\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-monitoring-logging.md#9\",\n \"claim\": \"Application Insights adaptive sampling: default 5 items/sec\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/ai-services-monitoring-logging.md#10\",\n \"claim\": \"Copilot Studio-bottar kan kobles til Application Insights for conversation analytics, LUIS intent recognition performance og QnA Maker query latency\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/ai-services-monitoring-logging.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-image-analysis.md", + "claim_count": 14, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-image-analysis.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-image-analysis.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#1\",\n \"claim\": \"Image Analysis 4.0 er generelt tilgjengelig (GA) siden november 2023, bygget på Florence foundation model fra Microsoft Research\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/whats-new\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#2\",\n \"claim\": \"Image Analysis 4.0 støtter synkron OCR, dense captions (opptil 10 regioner i bildet), people detection og smart crop\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/overview-image-analysis\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#3\",\n \"claim\": \"Visual features i Image Analysis 4.0 med regionsrestriksjoner: Caption, Dense Captions og Smart Crops kun i visse Azure-regioner; Tags, Objects, People og Read (OCR) i alle regioner\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/overview-image-analysis\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#4\",\n \"claim\": \"Regioner med full funksjonalitet (Caption/Dense Captions/Smart Crop): East US, West US, France Central, North Europe, West Europe, Southeast Asia, East Asia, Korea Central\",\n \"claim_type\": \"region\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/concept-describe-images-40\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#5\",\n \"claim\": \"Content moderation (adult/racy/gory) finnes i Image Analysis 3.2 (fortsatt støttet), men er fjernet i Image Analysis 4.0 — Azure AI Content Safety anbefales i stedet\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/overview-image-analysis\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#6\",\n \"claim\": \"Multimodal embeddings (4.0) gir 1024-dimensjonal vektor\",\n \"claim_type\": \"version\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#7\",\n \"claim\": \"Multilingual embedding-modell støtter 102 språk (2024-02-01 API); embeddings fra engelsk-modellen er ikke kompatible med multilingual-modellen\",\n \"claim_type\": \"version\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#8\",\n \"claim\": \"Image Analysis: maks bildestørrelse 20 MB\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/overview-image-analysis\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#9\",\n \"claim\": \"Rate limits: 10-20 requests/sekund per tier (opp til 20 requests/sekund per Standard tier)\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#10\",\n \"claim\": \"Custom Vision krever trening med minimum 30 bilder per tag\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#11\",\n \"claim\": \"Objektdeteksjon: objekter mindre enn 5 % av bildestørrelsen detekteres ikke\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/concept-object-detection-40\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#12\",\n \"claim\": \"Power Automate Azure AI Vision connector støtter Image Analysis 3.2, ikke 4.0 (per feb 2026) — HTTP action kreves for 4.0-features\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#13\",\n \"claim\": \"Azure AI Vision er tilgjengelig i Norway East og Norway West\",\n \"claim_type\": \"region\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-image-analysis.md#14\",\n \"claim\": \"Azure AI Vision Free tier (F0): 0-5000 transaksjoner/måned gratis; Standard tier heter S1\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-image-analysis.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-ocr-processing.md", + "claim_count": 13, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-ocr-processing.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-ocr-processing.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#1\",\n \"claim\": \"Azure AI Content Understanding er GA (per 2026-04) og anbefales for OCR kombinert med semantisk analyse/layout/tabeller\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#2\",\n \"claim\": \"Azure Vision v4.0 Read: synkron API, GA, del av Image Analysis 4.0 | Azure Vision v3.2 Read: asynkron, GA men legacy, ingen videre oppdateringer etter v3.2 | Document Intelligence Read: asynkron, GA, optimalisert for teksttunge dokumenter; begge editions bruker samme OCR-motor\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/overview-ocr\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#3\",\n \"claim\": \"OCR trykt tekst støtter flere språk inkludert engelsk, fransk, tysk, italiensk, portugisisk, spansk, kinesisk, japansk, koreansk, russisk, arabisk, hindi; skriftsystemer inkl. latinske, kyrilliske, arabiske og devanagari-tegnsett\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/overview-ocr\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#4\",\n \"claim\": \"OCR håndskrift støttes for engelsk, kinesisk (forenklet), fransk, tysk, italiensk, japansk, koreansk, portugisisk og spansk; handwritten classification kun for latinsk alfabet\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/overview-ocr\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#5\",\n \"claim\": \"OCR input-krav: filformater JPEG/PNG/BMP/PDF/TIFF | maks filstørrelse 500 MB (4 MB gratisnivå) | dimensjoner min 50x50 til maks 10000x10000 piksler | PDF/TIFF opptil 2000 sider (kun 2 første sider på gratisnivå) | minimum teksthøyde 12 piksler for 1024x768-bilde\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/how-to/call-read-api\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#6\",\n \"claim\": \"API-alternativer: Image Analysis 4.0 Read synkron med input JPEG/PNG/BMP/GIF | Document Intelligence Read asynkron med input PDF/TIFF/JPEG/PNG/BMP/Office-filer | Azure Vision v3.2 Read asynkron legacy med input JPEG/PNG/BMP/PDF/TIFF\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/overview-ocr\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#7\",\n \"claim\": \"Azure AI Search Image Analysis Skill bruker v3.2 API (legacy); for v4.0-funksjonalitet kreves custom Web API skill\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/cognitive-search-skill-image-analysis\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#8\",\n \"claim\": \"AI Builder tilbyr en Text Recognition prebuilt model som bruker Azure Vision OCR under panseret (Power Automate-integrasjon)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#9\",\n \"claim\": \"SynapseML tilbyr en ReadImage-transformator for OCR i Spark-pipelines (Microsoft Fabric/Synapse)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/synapse-analytics/machine-learning/tutorial-computer-vision-use-mmlspark\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#10\",\n \"claim\": \"OCR data retention: input-bilder og ekstrahert tekst lagres midlertidig i 48 timer (operation-location URL), deretter slettet automatisk; ingen permanent lagring av kundedata\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/computer-vision/ocr-data-privacy-security\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#11\",\n \"claim\": \"OCR-data krypteres under transit med TLS 1.2 og ved hvile (Azure Storage encryption)\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/computer-vision/ocr-data-privacy-security\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#12\",\n \"claim\": \"Gratisnivåer: Azure Vision v4.0 Read OCR 5000 transaksjoner/måned gratis (Standard S1) | Document Intelligence Read 500 sider/måned gratis (Standard S0)\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/azure-ai-vision-ocr-processing.md#13\",\n \"claim\": \"Read Docker container finnes for on-premises deployment (compliance/air-gapped-krav)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/azure-ai-vision-ocr-processing.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/content-understanding-multimodal-analysis.md", + "claim_count": 12, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/content-understanding-multimodal-analysis.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/content-understanding-multimodal-analysis.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#1\",\n \"claim\": \"Content Understanding status: Preview (GA for kjernefunksjoner, Limited Access for face description)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-understanding/video/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#2\",\n \"claim\": \"Tekniske begrensninger video/audio: frame sampling ~1 FPS | frame resolution 512×512 px resize | kun tale transkriberes (musikk/lydeffekter/bakgrunnsstøy ignoreres)\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-understanding/video/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#3\",\n \"claim\": \"Prebuilt analyzers: prebuilt-videoSearch (RAG-optimalisert, WEBVTT-transcript + keyframes) | prebuilt-videoAnalysis (generell video-metadata) | prebuilt-documentSearch (markdown for RAG)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-understanding/video/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#4\",\n \"claim\": \"audioVisual-elementer som krever returnDetails:true: transcript phrases og camera shots; markdown content, contents collection, timing, key frames og field extraction krever det ikke; key frames og camera shots gjelder kun video\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-understanding/video/elements\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#5\",\n \"claim\": \"Keyframes samples uniformt fra hver camera shot, minimum én per shot, deterministisk utvalg; cameraShotTimesMs angir startpunkt per shot (første shot starter alltid ved 0 ms); detekterer abrupte og gradvise overganger\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-understanding/video/elements\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#6\",\n \"claim\": \"Content Safety multimodal analysis er begrenset til regionene East US og West Europe\",\n \"claim_type\": \"region\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#7\",\n \"claim\": \"Content Understanding vs Video Indexer: CU har ikke face recognition (kun face description via Limited Access) og ikke real-time analyse; Video Indexer har celebrity + custom faces, live video streaming, observed people tracking (bounding boxes); CU prises token-basert, Video Indexer page/minute-basert\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-video-indexer/insights-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#8\",\n \"claim\": \"Content Understanding er core Foundry-tjeneste tilgjengelig via Microsoft Foundry portal, Foundry SDK (Python, .NET) med ContentUnderstandingClient-klasse, og REST API POST /contentunderstanding/analyzers/{analyzerId}:analyze\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/python/api/overview/azure/ai-contentunderstanding-readme\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#9\",\n \"claim\": \"Content Understanding kan integreres som skill i Azure AI Search indexer pipeline; CU-skill gir cross-page text location metadata, cross-page table extraction og semantic chunking (vs Document Extraction/Document Layout skills); støttede formater PDF, DOCX, XLSX, PPTX\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/multimodal-search-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#10\",\n \"claim\": \"Face description er Limited Access-feature: krever Azure support request + justification, Microsoft vurderer use case før godkjenning\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/content-understanding/data-privacy\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#11\",\n \"claim\": \"Free tier er ikke tilgjengelig for Content Understanding; krever betalt Azure subscription\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/content-understanding-multimodal-analysis.md#12\",\n \"claim\": \"Webhooks for async completion er preview-feature, ikke GA (per Feb 2026)\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/content-understanding-multimodal-analysis.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-custom-models.md", + "claim_count": 11, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-custom-models.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-custom-models.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-custom-models.md#1\",\n \"claim\": \"Document Intelligence custom models status: GA (v4.0 API, api-version 2024-11-30)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/train/custom-model?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-custom-models.md#2\",\n \"claim\": \"v4.0 (GA) ga custom neural models støtte for signaturdeteksjon (min. 5 samples), overlappende felter (complete/partial overlap) og tabell-/rad-/celle-konfidensscoring\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/train/custom-neural?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-custom-models.md#3\",\n \"claim\": \"Custom neural models: opptil 50 000 siders treningsdata, trening opptil 10 timer (10 gratis timer per måned, deretter betalt trening); minimum 5 dokumenter for å komme i gang\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/train/custom-neural?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-custom-models.md#4\",\n \"claim\": \"Treningstid: custom template 1-5 minutter | custom neural 30 min-12 timer\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/train/custom-model?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-custom-models.md#5\",\n \"claim\": \"Treningskrav: min 5 dokumenter (begge typer) | maks treningssider 500 (template) / 50 000 (neural) | maks treningsstørrelse 50 MB (template) / 1 GB (neural) | filformater PDF, JPEG/JPG, PNG, BMP, TIFF, HEIF | maks 2 000 sider per dokument (F0: 2 sider) | maks filstørrelse analyse S0 500 MB / F0 4 MB | bildedimensjoner 50×50 til 10 000×10 000 px\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/how-to-guides/build-a-custom-model?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-custom-models.md#6\",\n \"claim\": \"Treningsbudsjett: v4.0 (2024-11-30) 10 gratis timer/måned, maks 10 timer per build, faktureres for faktisk tid (min. 30 min per jobb, krever eksplisitt maxTrainingHours); v3.1/v3.0: 20 byggeoperasjoner/måned, maks 30 minutter per build, ingen betalt trening\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/train/custom-model?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-custom-models.md#7\",\n \"claim\": \"Composed models: kombiner opptil 200 custom models til én modell-ID; Document Intelligence klassifiserer dokumentet automatisk og velger best match\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/how-to-guides/compose-custom-models?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-custom-models.md#8\",\n \"claim\": \"Custom classifier i v4.0 støtter Office-formater (DOCX, XLSX, PPTX) og dokumentsplitting (én fil → mange dokumenter); min. 5 samples per klasse\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/train/custom-classifier?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-custom-models.md#9\",\n \"claim\": \"AI Builder custom models (Document Processing) støtter kun template models, ikke neural (per januar 2026)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-custom-models.md#10\",\n \"claim\": \"Custom neural models kan kun trenes i utvalgte regioner (inkl. West Europe, ikke Norway East); modeller kan kopieres til andre regioner (f.eks. Norway East) for produksjon\",\n \"claim_type\": \"region\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/train/custom-neural?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-custom-models.md#11\",\n \"claim\": \"Custom models lagres i 90 dager uten kostnad\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-custom-models.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-prebuilt-models.md", + "claim_count": 12, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-prebuilt-models.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-prebuilt-models.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#1\",\n \"claim\": \"Azure AI Content Understanding er fullt GA (per 2026-04) og anbefalt startpunkt for de fleste IDP-scenarier; Document Intelligence anbefales for spesifikke prebuilt-skjemaer (regnskap, ID-dokumenter, kvitteringer)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/content-understanding/choosing-right-ai-tool#azure-document-intelligence\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#2\",\n \"claim\": \"Prebuilt-modellene støtter 27 språk\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/document-intelligence/language-support/prebuilt?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#3\",\n \"claim\": \"v4.0 (GA: 2024-11-30) introduserte nye felt: ReceiptType, TaxDetails og VAT-ekstraksjon for hotellkvitteringer\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/document-intelligence/prebuilt/receipt?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#4\",\n \"claim\": \"Financial Services prebuilt-modeller: prebuilt-invoice | prebuilt-receipt | prebuilt-bankStatement | prebuilt-creditCard | prebuilt-check | prebuilt-contract | prebuilt-payStub.us\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/document-intelligence/model-overview?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#5\",\n \"claim\": \"Identity & Tax prebuilt-modeller: prebuilt-idDocument | prebuilt-healthInsuranceCard.us | prebuilt-marriageCertificate | prebuilt-tax.us.w2 | prebuilt-tax.us.1098 | prebuilt-tax.us.1099 | prebuilt-tax.us.1040 | prebuilt-tax.us (unified)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/document-intelligence/model-overview?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#6\",\n \"claim\": \"US Mortgage prebuilt-modeller: prebuilt-mortgage.us.1003 | prebuilt-mortgage.us.1004 | prebuilt-mortgage.us.1005 | prebuilt-mortgage.us.1008 | prebuilt-mortgage.us.disclosure\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/document-intelligence/model-overview?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#7\",\n \"claim\": \"Grunnleggende modeller: prebuilt-read (OCR: tekst, linjer, ord, språkdeteksjon) | prebuilt-layout (tabeller, selection marks, seksjoner, valgfrie key-value pairs) | prebuilt-document (key-value pairs, tabeller, selection marks)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/document-intelligence/model-overview?view=doc-intel-4.0.0\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#8\",\n \"claim\": \"AI Builder bruker Document Intelligence v3.1 (ikke alltid v4.0); Premium-lisens påkrevd for AI Builder; 1M AI Builder credits inkludert i visse Power Apps/Automate-lisenser\",\n \"claim_type\": \"version\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#9\",\n \"claim\": \"Logic Apps har native connector 'Azure AI Document Intelligence' med full API v4.0-støtte og managed identity authentication\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#10\",\n \"claim\": \"Free (F0): 500 sider/måned, 2 sider per dokument, 20 calls/min | Standard (S0): 2 000 sider per dokument, 15 TPS\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#11\",\n \"claim\": \"Standard 30-dagers oppbevaring av dokumenter i Document Intelligence (kan slettes umiddelbart etter prosessering)\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/document-intelligence-prebuilt-models.md#12\",\n \"claim\": \"Custom models utløper (slutter å virke) etter 12–24 måneder; krever retraining-schedule\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/document-intelligence-prebuilt-models.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/language-services-custom-text-classification.md", + "claim_count": 12, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/language-services-custom-text-classification.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/language-services-custom-text-classification.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#1\",\n \"claim\": \"Custom Text Classification/Custom NER status: GA — avvikles 31. mars 2029\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/language-service/custom-text-classification/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#2\",\n \"claim\": \"Custom Text Classification og Custom NER avvikles 2029-03-31; anbefalt migrering til Microsoft Foundry-modeller (prompt-basert klassifisering og NER med GPT-4o eller GPT-4.1)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/language-service/custom-text-classification/how-to/migrate-azure-openai\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#3\",\n \"claim\": \"Custom Text Classification støtter to prosjekttyper: Single label classification (ett dokument, én kategori) og Multi label classification (ett dokument, flere kategorier)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/custom-text-classification/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#4\",\n \"claim\": \"Tjenestene er tilgjengelige via Microsoft Foundry portal (ai.azure.com) og via REST API/SDK-er for Python, C#, Java og JavaScript\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/custom-text-classification/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#5\",\n \"claim\": \"Standard NER dekker 18+ entity-typer out-of-the-box (person, location, org, quantity, datetime, etc.)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#6\",\n \"claim\": \"Language resource må ha 'Custom text classification & custom named entity recognition'-feature aktivert, som krever storage account-tilkobling (og managed identity)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/custom-named-entity-recognition/how-to/create-project\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#7\",\n \"claim\": \"Custom Text Classification kan ikke integreres direkte i M365 Copilot (Copilot bruker forhåndstrente modeller); workaround via Copilot Studio-bot eller Power Automate-flow\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#8\",\n \"claim\": \"Tekst sendt til runtime-API logges ikke, men respons caches i 15 min (kan deaktiveres med loggingOptOut: true)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#9\",\n \"claim\": \"'Text record' = inntil 1000 characters; lengre tekster teller som flere records (2500 characters = 3 records)\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#10\",\n \"claim\": \"Free tier (F0): 5000 text records per måned for Prediction API | trening ubegrenset | maks 1 deployment per prosjekt\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#11\",\n \"claim\": \"Batch API: opptil 10 dokumenter per request\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-custom-text-classification.md#12\",\n \"claim\": \"Custom Classification støtter 100+ språk (multilingual-funksjonalitet)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/custom-text-classification/language-support\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/language-services-custom-text-classification.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/language-services-question-answering.md", + "claim_count": 15, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/language-services-question-answering.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/language-services-question-answering.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#1\",\n \"claim\": \"Custom Question Answering status: GA — avvikles 31. mars 2029\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/language-service/question-answering/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#2\",\n \"claim\": \"CQA avvikles 2029-03-31; anbefalt migrering til Microsoft Foundry-baserte løsninger: Agentic Retrieval, Azure AI Search + GPT-4o RAG-pipeline, eller AI Foundry Knowledge Retrieval\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/language-service/question-answering/how-to/migrate-qnamaker-to-question-answering\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#3\",\n \"claim\": \"CQA er etterfølgeren til QnA Maker, som ble retired mars 2025; migrering via export/import av TSV-filer\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/language-service/question-answering/how-to/migrate-qnamaker-to-question-answering\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#4\",\n \"claim\": \"CQA støtter 53 språk inkludert norsk; transformer-baserte rankeringsmodeller\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/question-answering/language-support\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#5\",\n \"claim\": \"CQA krever to ressurser: Language resource (S-tier anbefalt for produksjon) + Azure AI Search (S1-tier for 10 TPS throughput-cap)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/question-answering/concepts/azure-resources\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#6\",\n \"claim\": \"Index-allokering i Azure AI Search: N-1 indexes for single-language-prosjekter (14 prosjekter på tier med 15 indexes), N/2 for multi-language (7 prosjekter); siste index reservert for authoring/testing\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/question-answering/concepts/azure-resources\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#7\",\n \"claim\": \"SDK-er: .NET Azure.AI.Language.QuestionAnswering (runtime, authoring-pakke tilgjengelig) og Python azure-ai-language-questionanswering (runtime og authoring)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/question-answering/quickstart/sdk\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#8\",\n \"claim\": \"Ingen hard limit på antall dokumenter per prosjekt; maks dokumentstørrelse 25 GB per S-tier prosjekt\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#9\",\n \"claim\": \"Chit-chat personality datasets: Professional | Friendly | Witty | Caring | Enthusiastic\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#10\",\n \"claim\": \"QnA Maker native integration i Copilot Studio (Power Virtual Agents) er deprecated; må bruke REST API-tilkobling\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#11\",\n \"claim\": \"CQA er ikke en native data source for M365 Copilot (ingen direkte integrasjon)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#12\",\n \"claim\": \"Cross-region replication er ikke tillatt i CQA; Azure AI Search må være i samme region som Language resource\",\n \"claim_type\": \"region\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/question-answering/concepts/azure-resources\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#13\",\n \"claim\": \"Azure AI Search-kapasitet for CQA: Free 3 indexes / 50 MB / 2 KBs | Basic 15 indexes / 2 GB / 14 KBs (single-language) | S1 50 indexes / 25 GB / 49 KBs\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/question-answering/concepts/azure-resources\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#14\",\n \"claim\": \"Language resource med CQA: Free (F0) 5000 text records/måned | Standard (S) ubegrensede hosted calls\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-question-answering.md#15\",\n \"claim\": \"CQA har hard cap på 10 TPS (transactions per second); overskridelse gir HTTP 429\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/question-answering/concepts/azure-resources\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/language-services-question-answering.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/language-services-text-analytics.md", + "claim_count": 10, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/language-services-text-analytics.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/language-services-text-analytics.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-text-analytics.md#1\",\n \"claim\": \"Sentiment Analysis, Opinion Mining og Custom Text Classification avvikles 2029-03-31; PII Detection, Key Phrase Extraction og Language Detection er ikke berørt; migrasjonssti er Microsoft Foundry-modeller\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/sentiment-opinion-mining/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-text-analytics.md#2\",\n \"claim\": \"Azure AI Language text analytics-kjernekapabiliteter: Sentiment Analysis (med opinion mining) | Key Phrase Extraction | Named Entity Recognition | Language Detection; tilgjengelig som REST API, SDK (C#, Java, Python, JavaScript) og Docker-containere\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/sentiment-opinion-mining/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-text-analytics.md#3\",\n \"claim\": \"Tjenesten er stateless (ingen data lagres); batch-resultater er tilgjengelige i 24 timer før automatisk sletting\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-text-analytics.md#4\",\n \"claim\": \"Key Phrase Extraction støtter 94 språk (inkl. norsk, samisk, finsk, svensk, dansk); introduksjonen sier 94+ språk\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/key-phrase-extraction/language-support\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-text-analytics.md#5\",\n \"claim\": \"NER-entitetskategorier: Person/PersonType | Organization (Medical/Sports/StockExchange) | Location (City, CountryRegion, State, GPE, Airport, Continent) | DateTime | Quantity (Number, Percentage, Currency, Age, Temperature, Speed, Weight, Volume, Area, Length) | Event | Contact (Email, PhoneNumber, URL, IpAddress, Address) | Product/ComputingProduct | Skill, Information\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-service/named-entity-recognition/concepts/named-entity-categories\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-text-analytics.md#6\",\n \"claim\": \"Docker-containere støtter Sentiment, Language Detection, Key Phrase, Custom NER og Text Analytics for Health\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-text-analytics.md#7\",\n \"claim\": \"Docker-containere krever Standard S tier; Free F0 tier støttes ikke for containere\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-text-analytics.md#8\",\n \"claim\": \"Maks 5000 tegn per text record; større dokumenter må splittes\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-text-analytics.md#9\",\n \"claim\": \"Azure Norway-datasentre (Oslo, Stavanger) støtter ikke Language Services per 2026-02; nærmeste regioner er West Europe (Nederland) og North Europe (Irland)\",\n \"claim_type\": \"region\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/language-services-text-analytics.md#10\",\n \"claim\": \"Free F0 tier gir 5000 gratis text records per måned\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/language-services-text-analytics.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/speech-services-speaker-recognition.md", + "claim_count": 8, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/speech-services-speaker-recognition.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/speech-services-speaker-recognition.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speaker-recognition.md#1\",\n \"claim\": \"Speaker Recognition (alle features) er Limited Access: registrering kreves, kun Microsoft-managed customers, personlige e-postadresser avvises, godkjenning tar 5-10 virkedager\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/cognitive-services-limited-access\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speaker-recognition.md#2\",\n \"claim\": \"Text-dependent verification bruker 10 forhåndsdefinerte engelske passphrases; similarity threshold >= 0.5 (kombinert voice + tekst); text-independent threshold >= 0.5 (kun voice)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speaker-recognition-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speaker-recognition.md#3\",\n \"claim\": \"Speaker Identification: maks 50 kandidat-speakers per request\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speaker-recognition-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speaker-recognition.md#4\",\n \"claim\": \"Speaker Identification-respons: 1 identified ID + 5 top-ranked IDs med scores; default threshold 0.5 (kan overstyres); returnerer \\\"0\\\"-string hvis ingen score >= 0.5\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/rest/api/speakerrecognition/\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speaker-recognition.md#5\",\n \"claim\": \"Speaker Recognition-API-ene er ikke designet for liveness detection (levende person vs. opptak/imitasjon); replay attack-mitigering må implementeres separat\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speaker-recognition-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speaker-recognition.md#6\",\n \"claim\": \"Speaker Recognition/Speech-ressurs tilgjengelig i Norway East (Oslo) og West Europe; voice signatures lagres i samme region som Speech resource\",\n \"claim_type\": \"region\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speaker-recognition.md#7\",\n \"claim\": \"Teams Intelligent Speaker krever Teams Rooms Pro (ikke Standard/Basic); Copilot speaker attribution krever Teams Premium eller Copilot-lisens\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/microsoftteams/rooms/voice-recognition\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speaker-recognition.md#8\",\n \"claim\": \"Speech SDK er gratis, open source med MIT-lisens\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/speech-services-speaker-recognition.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/speech-services-speech-to-text.md", + "claim_count": 14, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/speech-services-speech-to-text.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/speech-services-speech-to-text.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speech-to-text.md#1\",\n \"claim\": \"Speech to text har tre hovedmodi: real-time transcription (live/streaming), fast transcription (synkron), batch transcription (asynkron)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-to-text\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speech-to-text.md#2\",\n \"claim\": \"Fast transcription: lydfiler maks 5 timer og maks 500 MB; output kun display form\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/fast-transcription-create\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speech-to-text.md#3\",\n \"claim\": \"Batch transcription: asynkron prosessering 30 min - 24 timer (best-effort scheduling); output JSON med lexical + display form; ubegrenset filstørrelse\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/batch-transcription\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speech-to-text.md#4\",\n \"claim\": \"Custom speech-modeller krever deployment til custom endpoint, bortsett fra ved batch transcription; modeller utløper etter en definert periode (model lifecycle)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/custom-speech-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speech-to-text.md#5\",\n \"claim\": \"Speaker diarization: maksimalt 2 kanaler hvis diarization er aktivert; støttes ikke på tvers av flere kanaler samtidig\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-to-text\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speech-to-text.md#6\",\n \"claim\": \"Multi-lingual transcription er preview og støtter 15 språk: de-DE, en-AU, en-CA, en-GB, en-IN, en-US, es-ES, es-MX, fr-CA, fr-FR, it-IT, ja-JP, ko-KR, pt-BR, zh-CN\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-to-text\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speech-to-text.md#7\",\n \"claim\": \"Phrase list er tilgjengelig i fast transcription API version 2025-10-15\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/fast-transcription-create\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speech-to-text.md#8\",\n \"claim\": \"Støttede lydformater: WAV, MP3, OPUS/OGG, FLAC, WMA, AAC, ALAW (WAV), MULAW (WAV), AMR, WebM, SPEEX\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/batch-transcription\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speech-to-text.md#9\",\n \"claim\": \"Speech SDK støtter C#, Python, Java, JavaScript, C++, Go; Speech CLI heter spx; Speech Studio på speech.microsoft.com\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-to-text\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speech-to-text.md#10\",\n \"claim\": \"Batch transcription API kan bruke Whisper model; Azure OpenAI tilbyr også Whisper med andre pricing og capabilities enn Azure Speech batch transcription\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/batch-transcription\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speech-to-text.md#11\",\n \"claim\": \"Batch best practices: send ~1000 filer per Transcription_Create request; poll status maks 1 gang per minutt (ideelt hvert 5-10 min)\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/batch-transcription\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speech-to-text.md#12\",\n \"claim\": \"Custom models bygget på base models fra oktober 2023 eller senere koster penger å trene; tidligere base models er gratis å trene\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/custom-speech-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speech-to-text.md#13\",\n \"claim\": \"Azure Speech støtter West Europe og North Europe regions; audio og transkripsjondata kan lagres i EU\",\n \"claim_type\": \"region\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-speech-to-text.md#14\",\n \"claim\": \"Logging data i Microsoft-owned storage slettes etter 30 dager; customer-owned storage kontrolleres av kunden\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/speech-services-speech-to-text.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/speech-services-text-to-speech.md", + "claim_count": 14, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/speech-services-text-to-speech.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/speech-services-text-to-speech.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-text-to-speech.md#1\",\n \"claim\": \"Azure Neural TTS og Custom Neural Voice er begge GA og aktivt vedlikeholdt per 2026-04; Custom Neural Voice Pro tilbyr ytterligere enterprise-tilpasning\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/custom-neural-voice\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-text-to-speech.md#2\",\n \"claim\": \"TTS tilbyr over 400 stemmer på 140+ språk og dialekter\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-text-to-speech.md#3\",\n \"claim\": \"Hvert standard neural voice-modell er tilgjengelig i 24 kHz og høy-fidelitet 48 kHz; output kan opp-/ned-samples\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/text-to-speech\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-text-to-speech.md#4\",\n \"claim\": \"en-US-AvaMultilingualNeural støtter 91 locales\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-text-to-speech.md#5\",\n \"claim\": \"OpenAI TTS voices er tilgjengelig via Azure Speech i regionene North Central US og Sweden Central\",\n \"claim_type\": \"region\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/text-to-speech\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-text-to-speech.md#6\",\n \"claim\": \"Custom Neural Voice er en Limited Access-funksjon som krever søknad (intake form aka.ms/customneural) og godkjenning fra Microsoft\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/custom-neural-voice\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-text-to-speech.md#7\",\n \"claim\": \"Batch Synthesis REST API er preview; for lange lydfiler (>10 min); real-time synthesis har <10 min audio-lengde og SDK-støtte, batch er REST API only\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/batch-synthesis\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-text-to-speech.md#8\",\n \"claim\": \"Custom Neural Voice training: 20-40 compute hours training (single-style), multi-style ca. 90 compute hours, fakturerings-cap 96 compute hours; 48kHz output etter engine upgrade\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/custom-neural-voice\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-text-to-speech.md#9\",\n \"claim\": \"Billable characters: alle tegn teller inkl. mellomrom/tegnsetting; SSML markup teller unntatt og tags; kinesiske tegn (også kanji/hanja/hanzi) = 2 billable characters\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/text-to-speech\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-text-to-speech.md#10\",\n \"claim\": \"TTS er ikke native i M365 Copilot per januar 2026; integreres via custom connectors (Power Automate)\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-text-to-speech.md#11\",\n \"claim\": \"Azure OpenAI TTS API støtter tts-1 og tts-1-hd modeller med stemmene alloy, echo, fable, onyx, nova, shimmer\",\n \"claim_type\": \"version\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-text-to-speech.md#12\",\n \"claim\": \"Voice Live API kombinerer STT, LLM og TTS i én WebSocket-forbindelse; støtter real-time avatar synthesis\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-text-to-speech.md#13\",\n \"claim\": \"Norsk bokmål (nb-NO) har stemmene nb-NO-PernilleNeural (F) og nb-NO-FinnNeural (M); nynorsk og samisk er ikke støttet\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/speech-services-text-to-speech.md#14\",\n \"claim\": \"Free tier (F0): 5 audio requests/month og 0.5M characters/month; S0 tier for produksjon\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://azure.microsoft.com/pricing/details/cognitive-services/speech-services/\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/speech-services-text-to-speech.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/translator-custom-neural-models.md", + "claim_count": 11, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/translator-custom-neural-models.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/translator-custom-neural-models.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-custom-neural-models.md#1\",\n \"claim\": \"Custom Translator (Azure Translator) har status GA\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/translator/custom-translator/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-custom-neural-models.md#2\",\n \"claim\": \"Custom Translator-modell gir typisk 5-10 BLEU-poeng forbedring sammenlignet med baseline-modellen\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/translator/custom-translator/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-custom-neural-models.md#3\",\n \"claim\": \"Training krever minimum 10,000 parallelle setninger (anbefalt 50,000+); tuning og testing anbefalt 2,500 setninger hver (auto-genereres hvis ikke levert)\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/translator/custom-translator/concepts/model-training\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-custom-neural-models.md#4\",\n \"claim\": \"Dictionary-only training er mulig uten 10,000 parallelle setninger, men gir ingen BLEU-score; phrase dictionary er case-sensitive 100% match, sentence dictionary case-insensitive\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/translator/custom-translator/concepts/model-training\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-custom-neural-models.md#5\",\n \"claim\": \"Støttede filformater: TMX, XLIFF, XLF, DOCX, LCL, HTML, HTM, TXT (UTF-8/UTF-16), PDF, XLSX, ALIGN (pre-aligned); arkiver ZIP, GZ, TGZ; ZIP-navnekonvensjon {document_name}_{language_code}\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/translator/custom-translator/concepts/document-formats-naming-convention\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-custom-neural-models.md#6\",\n \"claim\": \"Custom-modeller brukes via Translator Text API v3 (api-version=3.0) med category-parameter mot api.cognitive.microsofttranslator.com; CategoryID-format {WorkspaceID}-{ProjectLabel}-{CategoryCode}\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/translator/custom-translator/how-to/translate-with-custom-model\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-custom-neural-models.md#7\",\n \"claim\": \"Custom Translator er tilgjengelig i Microsoft Foundry (klassisk portal) for no-code workflows med document upload, BLEU-sammenligning og deployment management\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-custom-neural-models.md#8\",\n \"claim\": \"Document Translation (asynkron batch) kan bruke custom-modeller med samme CategoryID-parameter; krever Azure Blob Storage for input/output; Speech Service kan også bruke Custom Translator CategoryID for speech translation\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-custom-neural-models.md#9\",\n \"claim\": \"Microsoft Translator har EU Data Boundary-sertifisering; baseline NMT-modeller er trent på global data, custom-modeller bruker kun kundens data\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-custom-neural-models.md#10\",\n \"claim\": \"Custom Translator støttes ikke på Free (F0) tier (kun baseline-modeller); Standard (S1) gir full støtte med ubegrenset antall workspaces, projects og modeller\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-custom-neural-models.md#11\",\n \"claim\": \"Custom Translator støtter 60+ språkpar; språkparet må inkludere engelsk som source eller target\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/language-support\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/translator-custom-neural-models.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/azure-ai-services/translator-document-translation.md", + "claim_count": 16, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/azure-ai-services/translator-document-translation.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/azure-ai-services/translator-document-translation.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#1\",\n \"claim\": \"Azure Translator støtter tekst- og dokumentoversettelse på tvers av over 135 språk og dialekter\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/translator/document-translation/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#2\",\n \"claim\": \"Batch document translation støtter bildefilformater (.jpeg, .png, .bmp, .webp) — nyhet desember 2025, preview (API-versjon 2025-12-01-preview)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/translator/document-translation/reference/start-batch-translation#translate-image-files\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#3\",\n \"claim\": \"Støttede batch-dokumentformater: .docx|.xlsx|.pptx|.msg|.pdf (OCR for scannede)|.html|.htm|.csv|.tsv|.mhtml|.md|.xlf|.odt|.ods|.odp\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/translator/document-translation/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#4\",\n \"claim\": \"Legacy-formater .doc, .xls, .ppt konverteres automatisk til .docx, .xlsx, .pptx ved oversettelse\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/translator/document-translation/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#5\",\n \"claim\": \"Funksjonsmatrise batch vs sync: multi-file translation, large file support, image text translation og multi-language docs kun batch; immediate response kun sync; custom glossaries, custom models og auto language detect i begge\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/translator/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#6\",\n \"claim\": \"Oversettelse av bildetekst i .docx/.pptx (batch) krever Azure AI Services multi-service resource (ikke standalone Translator) + parameter \\\"translateTextWithinImage\\\": true i options-feltet; tilleggskostnad basert på Azure Vision-prising\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/translator/document-translation/reference/start-batch-translation#translate-image-files\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#7\",\n \"claim\": \"Batch-response inkluderer feltene totalImageScansSucceeded og totalImageScansFailed for monitorering\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/translator/document-translation/reference/start-batch-translation#translate-image-files\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#8\",\n \"claim\": \"Synchronous single-file translation er egnet for filer < 40 MB (batch anbefalt for filstørrelse > 40 MB)\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/translator/service-limits#document-translation\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#9\",\n \"claim\": \"Document Translation tilgjengelig via Foundry (classic) portal som no-code playground; Foundry (new) støtter kun forhåndsdefinerte språk med sample-dokumenter (ikke egne filer)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#10\",\n \"claim\": \"Power Automate har Translator v3 connector som støtter både text og document translation\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#11\",\n \"claim\": \"Data residency: oversettelse prosesseres i nærmeste Azure-region (Global) eller spesifisert geografi (Americas, Asia Pacific, Europe); Europa-eksempler France Central og West Europe\",\n \"claim_type\": \"region\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/translator/document-translation/overview\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#12\",\n \"claim\": \"Microsoft lagrer ikke brukerdata etter oversettelse (dokumentert i Transparency Note)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/foundry/responsible-ai/translator/transparency-note\"\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#13\",\n \"claim\": \"Norway-region finnes ikke for Translator — bruk West Europe eller North Europe for geografisk nærhet\",\n \"claim_type\": \"region\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#14\",\n \"claim\": \"Translator Container (disconnected deployment) finnes for air-gapped miljøer\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#15\",\n \"claim\": \"Volume Discount Plans C2, C3, C4, D3 tilgjengelig for store volumer (commitment-basert)\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/azure-ai-services/translator-document-translation.md#16\",\n \"claim\": \"totalCharacterCharged returneres i response headers for cost tracking\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/ai-services/translator/document-translation/how-to-guides/use-rest-api-programmatically\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/azure-ai-services/translator-document-translation.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/data-engineering/data-factory-ai-pipelines.md", + "claim_count": 16, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/data-engineering/data-factory-ai-pipelines.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/data-engineering/data-factory-ai-pipelines.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#1\",\n \"claim\": \"Data Factory ekstraherer data fra 170+ kilder\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/fabric/data-factory/data-factory-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#2\",\n \"claim\": \"Copilot-drevet pipeline-design (natural language → pipeline-logikk) er kun tilgjengelig i Fabric Data Factory, ikke Azure Data Factory\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/fabric/data-factory/data-factory-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#3\",\n \"claim\": \"Status: GA for både Azure Data Factory og Fabric Data Factory\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/fabric/data-factory/data-factory-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#4\",\n \"claim\": \"Fabric Data Factory har native Azure Machine Learning Activity med batch endpoint + pipeline (v1) support\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#5\",\n \"claim\": \"Azure ML Update Resource activity støtter kun batch endpoints; online endpoints krever ARM templates eller Azure ML SDK via Notebook Activity\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/data-factory/transform-data-machine-learning-service\"\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#6\",\n \"claim\": \"Fabric Data Factory har kun AzureMLExecutePipeline (ikke AzureMLUpdateResource — update resource må gjøres via REST API)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/fabric/data-factory/migrate-planning-azure-data-factory\"\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#7\",\n \"claim\": \"Fabric tilbyr Apache Airflow Job (code-first Python DAGs) som alternativ til Pipeline, med Airflow providers apache-airflow-providers-microsoft-azure\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#8\",\n \"claim\": \"Copilot Studio har ikke native Data Factory connector (per Feb 2026)\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#9\",\n \"claim\": \"Data Factory kan ikke direkte kalle AI Builder-modeller (per Feb 2026)\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#10\",\n \"claim\": \"Azure Data Factory støtter Norway East/West regions (metadata lagres i region)\",\n \"claim_type\": \"region\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#11\",\n \"claim\": \"Fabric capacity er tilgjengelig i Norway West/East per Q1 2026 (OneLake-data residency følger capacity-region)\",\n \"claim_type\": \"region\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#12\",\n \"claim\": \"Fabric Data Factory CU-forbruk: Data movement (Copy) 1.5 CU/hour per intelligent throughput unit | Orchestration 0.0056 CU per activity run\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/fabric/data-factory/pricing-pipelines\"\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#13\",\n \"claim\": \"Fabric Capacity SKUs: F2 = 2 CU/sekund | F64 = 64 CU/sekund | F256 = 256 CU/sekund\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://azure.microsoft.com/en-us/pricing/details/microsoft-fabric/\"\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#14\",\n \"claim\": \"Azure Data Factory er ikke HIPAA BAA-compliant (per Feb 2026); Azure Synapse Pipelines er HIPAA-certified (samme teknologi)\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#15\",\n \"claim\": \"Microsoft tilbyr PowerShell-modulen Microsoft.FabricPipelineUpgrade for migrering av pipelines fra Azure Data Factory til Fabric Data Factory\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/fabric/data-factory/migrate-planning-azure-data-factory\"\n },\n {\n \"id\": \"ms-ai-engineering/data-engineering/data-factory-ai-pipelines.md#16\",\n \"claim\": \"Azure ML SDK v1 er deprecated per 2025-03-31 (data ingestion with ADF-dokumentasjonen gjelder SDK v1)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/machine-learning/how-to-data-ingest-adf\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/data-engineering/data-factory-ai-pipelines.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/mlops-genaiops/llm-evaluation-production.md", + "claim_count": 18, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/mlops-genaiops/llm-evaluation-production.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/mlops-genaiops/llm-evaluation-production.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#1\",\n \"claim\": \"Built-in LLM judges: Groundedness, Relevance, Coherence scorer på 1-5 skala; safety evaluators Violence, Sexual, Self-harm, Hate/Unfairness på 0-7 skala\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/how-to/develop/evaluate-sdk\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#2\",\n \"claim\": \"Agentic evaluators i Azure AI Evaluation: IntentResolutionEvaluator, TaskAdherenceEvaluator, ToolCallAccuracyEvaluator\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/how-to/develop/evaluate-sdk\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#3\",\n \"claim\": \"Max 20 scorers per experiment i MLflow production monitoring\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/databricks/mlflow3/genai/eval-monitor/\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#4\",\n \"claim\": \"MLflow 3 evaluation krever SDK mlflow[databricks]>=3.1\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/databricks/mlflow3/genai/eval-monitor/\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#5\",\n \"claim\": \"MLflow 3 built-in judges + ground truth-krav: RelevanceToQuery nei | RetrievalRelevance nei | RetrievalGroundedness nei | Safety nei | Correctness ja | RetrievalSufficiency ja | ToolCallCorrectness ja | ToolCallEfficiency nei | Guidelines nei | ExpectationsGuidelines nei (guidelines i expectations)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/databricks/mlflow3/genai/eval-monitor/concepts/scorers\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#6\",\n \"claim\": \"MLflow 3 multi-turn (conversation-level) judges: ConversationCompleteness, UserFrustration, KnowledgeRetention, ConversationalSafety, ConversationalGuidelines, ConversationalRoleAdherence, ConversationalToolCallEfficiency\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/databricks/mlflow3/genai/eval-monitor/concepts/scorers\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#7\",\n \"claim\": \"MLflow production monitoring bruker Databricks-hosted LLM judges; EU-workspaces bruker EU-hosted modeller; ingen prompts lagres hos Azure OpenAI (Abuse Monitoring opt-out)\",\n \"claim_type\": \"region\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/databricks/mlflow3/genai/eval-monitor/\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#8\",\n \"claim\": \"MLflow 3 erstattet Agent Evaluation SDK — migrer med mlflow.genai.*-funksjoner\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/databricks/mlflow3/genai/eval-monitor/\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#9\",\n \"claim\": \"Power Platform har ingen built-in continuous evaluation for LLM i production (gap per feb 2026)\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#10\",\n \"claim\": \"Copilot Studio: ingen out-of-the-box production evaluation; Microsoft roadmap (Q2 2026) inkluderer native integrasjon med Microsoft Foundry evaluation\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#11\",\n \"claim\": \"Databricks krever Premium eller Enterprise workspace tier for Unity Catalog governance\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#12\",\n \"claim\": \"azure.ai.evaluation tilbyr AIRedTeamingAgent for automatiserte adversarial scans (attack strategies: jailbreak, prompt_injection, bias_elicitation)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/python/api/overview/azure/ai-evaluation-readme\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#13\",\n \"claim\": \"azure.ai.evaluation tilbyr PIIMaskingPreprocessor med mask_types (email, phone, ssn, name)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/python/api/overview/azure/ai-evaluation-readme\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#14\",\n \"claim\": \"azure.ai.evaluation inkluderer ViolenceEvaluator, ProtectedMaterialEvaluator og CodeVulnerabilityEvaluator\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/python/api/overview/azure/ai-evaluation-readme\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#15\",\n \"claim\": \"Continuous evaluation for AI agents i Microsoft Foundry er preview\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/how-to/continuous-evaluation-agents\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#16\",\n \"claim\": \"Prompt Flow pensjoneres 2027-04-20 — migrer til Microsoft Agent Framework; Azure ML model monitoring for generative AI applications er preview\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/how-to-monitor-generative-ai-applications\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#17\",\n \"claim\": \"Agent Monitoring Dashboard i Microsoft Foundry er preview\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry/observability/how-to/how-to-monitor-agents-dashboard\"\n },\n {\n \"id\": \"ms-ai-engineering/mlops-genaiops/llm-evaluation-production.md#18\",\n \"claim\": \"Azure AI Evaluation SDK versjon v1.17.0 (per research-dato 2026-06-19)\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/python/api/overview/azure/ai-evaluation-readme\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/mlops-genaiops/llm-evaluation-production.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/rag-architecture/azure-ai-search-setup.md", + "claim_count": 15, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/rag-architecture/azure-ai-search-setup.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/rag-architecture/azure-ai-search-setup.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#1\",\n \"claim\": \"Tier-grenser: Free 10 000 docs / 3 indekser | Basic 15M docs / 15 indekser | S1 60M per partition / 50 indekser | S2 120M per partition / 200 indekser | S3 240M per partition / 200 indekser | L1/L2 120M/240M per partition / 10 indekser\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/search-sku-tier\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#2\",\n \"claim\": \"SLA 99.9% for 2+ replicas (query) og 3+ replicas (indexing + query); Basic kan møte SLA-krav med opptil 3 replicas\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/search-sku-tier\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#3\",\n \"claim\": \"Semantic Ranker er tilgjengelig på alle tiers (også Free) — gratis månedlig kvote (default) + standard pay-as-you-go-plan ved bruk utover kvoten\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/semantic-search-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#4\",\n \"claim\": \"Private Link (inbound) ikke tilgjengelig på Free tier; indexer-utgående tilkoblinger med skillsets støttes ikke på Free, Basic, S1 eller S3 HD\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/search-sku-tier\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#5\",\n \"claim\": \"Search services opprettet etter april 2024 (andre bølge mai 2024) i utvalgte regioner har større partisjoner og høyere vector-kvoter på samme billing-rate\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/search-sku-tier\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#6\",\n \"claim\": \"In-place tier-bytte støttes mellom Basic og Standard (S1/S2/S3); bytte til/fra Free, L1/L2 eller S3 HD krever ny service + migrering; S3 HD er hosting mode for mange små indekser med samme per-enhet-kostnad som S3\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/search-sku-tier\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#7\",\n \"claim\": \"Serverless (Developer) er ny consumption-basert prismodell (CU/h + GB/mnd) i Public Preview, kun i utvalgte regioner, støtter ikke migrering til/fra andre tiers\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/search-sku-tier\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#8\",\n \"claim\": \"Indexer (pull-modell) støttede kilder: Azure Blob Storage, Cosmos DB, SQL Database, SharePoint Online\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/search-indexer-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#9\",\n \"claim\": \"Free tier: ingen SLA, maks 10K docs, delte ressurser, kun én Free-service per subscription\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/search-sku-tier\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#10\",\n \"claim\": \"Prompt Flow (Microsoft Foundry + Azure ML) pensjoneres 2027-04-20 — migrer til Microsoft Agent Framework (MAF)\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/azure/machine-learning/prompt-flow/migrate-prompt-flow-to-agent-framework?view=azureml-api-2\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#11\",\n \"claim\": \"AI Foundry (tidligere AI Studio) har innebygget Vector Index-node i Prompt Flow med index_type Azure AI Search\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#12\",\n \"claim\": \"Azure AI Search støtter Norway East og Norway West regioner (data lagres kun i Norge, ingen geo-replikering utenfor EU/EEA)\",\n \"claim_type\": \"region\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#13\",\n \"claim\": \"AI Search throttler Push API ved >1000 docs per batch\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#14\",\n \"claim\": \"Hybrid search kjører full-text og vector search parallelt og merger resultater med Reciprocal Rank Fusion (RRF); støtter filtrering, faceting, sortering, scoring profiles og semantic ranking i én request\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/hybrid-search-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/azure-ai-search-setup.md#15\",\n \"claim\": \"maxTextRecallSize (preview) kontrollerer antall BM25-resultater inn til RRF-ranker: default 1000, max 10000\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/hybrid-search-overview\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/rag-architecture/azure-ai-search-setup.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-document-preprocessing.md", + "claim_count": 14, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/rag-architecture/rag-document-preprocessing.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/rag-architecture/rag-document-preprocessing.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-document-preprocessing.md#1\",\n \"claim\": \"Indexers kan kjøres on-demand eller på schedule ned til hvert 5. minutt\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/search-indexer-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-document-preprocessing.md#2\",\n \"claim\": \"Document Intelligence Layout-modellen støtter 309 trykte språk + 12 håndskrevne språk\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/concept/retrieval-augmented-generation\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-document-preprocessing.md#3\",\n \"claim\": \"Layout-modellen produserer Markdown-output, konverterer tabeller til Markdown, bevarer dokumentstruktur (headings, sections, paragraphs) og håndterer PDF-er, bilder, Office-filer og HTML i ett API-kall\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/concept/retrieval-augmented-generation\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-document-preprocessing.md#4\",\n \"claim\": \"Innebygde skills i Azure AI Search: OCR, Document Extraction, Text Merge, Text Split (størrelsesbasert), Azure Content Understanding Skill (semantisk layout-bevisst chunking, alternativ til Text Split for integrated vectorization), Document Intelligence Layout Skill, Language Detection\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/cognitive-search-concept-intro\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-document-preprocessing.md#5\",\n \"claim\": \"Støttede datakilder for indexers: Azure Blob Storage, Azure Data Lake Storage Gen2, Azure Cosmos DB, Azure SQL Database, SharePoint Online (preview), OneLake (Microsoft Fabric)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/search-indexer-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-document-preprocessing.md#6\",\n \"claim\": \"Azure OpenAI Embedding Skill med deploymentId text-embedding-3-large og dimensions 3072\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/search-how-to-integrated-vectorization\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-document-preprocessing.md#7\",\n \"claim\": \"Indexer batchSize har default 1\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/search-indexer-overview\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-document-preprocessing.md#8\",\n \"claim\": \"Document Intelligence v4.0 (2024-11-30) er GA\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/concept/retrieval-augmented-generation\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-document-preprocessing.md#9\",\n \"claim\": \"Azure AI Search lagringsgrenser: Free 50 MB / 3 indexes | Basic 2 GB / 15 indexes | Standard S1 25 GB / 50 indexes\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://azure.microsoft.com/en-us/pricing/details/search/\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-document-preprocessing.md#10\",\n \"claim\": \"Document Intelligence tiers: Free 500 sider/måned | Standard S0 pay-as-you-go\",\n \"claim_type\": \"sku\",\n \"evidence_url\": \"https://azure.microsoft.com/en-us/pricing/details/ai-document-intelligence/\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-document-preprocessing.md#11\",\n \"claim\": \"Azure OpenAI krever søknad om tilgang (compliance-vurdering)\",\n \"claim_type\": \"status\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-document-preprocessing.md#12\",\n \"claim\": \"imageAction: generateNormalizedImages er påkrevd for bildebehandling; maks 1000 bilder ekstraheres per dokument; kostnader påløper ved imageAction != none\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/cognitive-search-concept-image-scenarios\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-document-preprocessing.md#13\",\n \"claim\": \"Azure AI Search støtter tre tilnærminger til bildeinnhold i RAG: vektorisering (Azure Vision genererer søkbare vektorer), verbalisering (GenAI Prompt skill → LLM-tekstbeskrivelse), analyse/OCR (Image Analysis skill + OCR skill)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/search/cognitive-search-concept-image-scenarios\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-document-preprocessing.md#14\",\n \"claim\": \"Azure OpenAI i Norge: data residency i Norge (West Europe region for Azure AI Services)\",\n \"claim_type\": \"region\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/rag-architecture/rag-document-preprocessing.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-engineering/references/rag-architecture/rag-hallucination-mitigation.md", + "claim_count": 9, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-engineering/references/rag-architecture/rag-hallucination-mitigation.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-engineering/references/rag-architecture/rag-hallucination-mitigation.md`)\n\n[\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-hallucination-mitigation.md#1\",\n \"claim\": \"Groundedness-deteksjon: GA | groundedness correction/mitigating-feature: Preview, api-version 2024-09-15-preview\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-safety/concepts/groundedness\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-hallucination-mitigation.md#2\",\n \"claim\": \"Groundedness Detection-moduser: Non-Reasoning (rask, score uten forklaring) | Reasoning (detaljert forklaring av ungrounded segments) | Correction (auto-korrigering av ungrounded tekst)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-safety/concepts/groundedness\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-hallucination-mitigation.md#3\",\n \"claim\": \"Groundedness Detection API-parametre: domain=MEDICAL|GENERIC, task=QnA|Summarization, groundingSources (array), reasoning true/false, correction true\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-safety/concepts/groundedness\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-hallucination-mitigation.md#4\",\n \"claim\": \"Correction (auto-korrigering) krever Azure OpenAI GPT-4o som llmResource\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-safety/concepts/groundedness\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-hallucination-mitigation.md#5\",\n \"claim\": \"Azure OpenAI On Your Data grounding-parametre: strictness 1-5 | inScope true/false | top_n_documents\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-hallucination-mitigation.md#6\",\n \"claim\": \"Nødvendige ressurser: Azure OpenAI E0-tier | Azure AI Content Safety Standard tier (Groundedness Detection inkludert) | Azure AI Search S1 eller høyere for RAG-indexing\",\n \"claim_type\": \"sku\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-hallucination-mitigation.md#7\",\n \"claim\": \"Breaking change (oppdatert 2026-04): API-feltnavn endret correction → mitigating (deteksjons-modus) og correctedText → correctionText (output-felt)\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-safety/concepts/groundedness\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-hallucination-mitigation.md#8\",\n \"claim\": \"Groundedness Detection: kun engelsk tekst støttes; tilgjengelig i utvalgte Azure-regioner\",\n \"claim_type\": \"region\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/content-safety/concepts/groundedness\"\n },\n {\n \"id\": \"ms-ai-engineering/rag-architecture/rag-hallucination-mitigation.md#9\",\n \"claim\": \"Foundry GroundednessEvaluator (azure.ai.evaluation Python SDK) returnerer groundedness-score 1-5 + groundedness_reason\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-engineering/references/rag-architecture/rag-hallucination-mitigation.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-governance/references/monitoring-observability/alerting-strategies-escalation.md", + "claim_count": 12, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-governance/references/monitoring-observability/alerting-strategies-escalation.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-governance/references/monitoring-observability/alerting-strategies-escalation.md`)\n\n[\n {\n \"id\": \"ms-ai-governance/monitoring-observability/alerting-strategies-escalation.md#1\",\n \"claim\": \"Alert processing rules: filtre støtter opptil 5 verdier per filter (logisk OR); Suppression har høyere prioritet enn Apply action groups\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/alerts/alerts-processing-rules\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/alerting-strategies-escalation.md#2\",\n \"claim\": \"Email-notifications: opptil 1000 mottakere per action group; rate limit 1000 emails per hour per action group\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/alerts/action-groups#service-limits-for-notifications\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/alerting-strategies-escalation.md#3\",\n \"claim\": \"SMS: 1 SMS per 5 min per telefonnummer | Voice call: 1 call per 5 min per telefonnummer\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/alerts/action-groups#service-limits-for-notifications\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/alerting-strategies-escalation.md#4\",\n \"claim\": \"Webhook retry: 5 retries med intervaller 5s/20s/5s/40s/5s + 15-min cooldown\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/alerts/action-groups#webhook\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/alerting-strategies-escalation.md#5\",\n \"claim\": \"Event Hub-notification: supports Private Link og NSP; global resilience regional\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/alerts/action-groups\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/alerting-strategies-escalation.md#6\",\n \"claim\": \"Opptil 5 action groups per alert rule; action groups kjøres concurrent uten bestemt rekkefølge; samme action group kan brukes av flere alert rules\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/alerts/action-groups\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/alerting-strategies-escalation.md#7\",\n \"claim\": \"Managed identity-støtte per action type: Automation Runbook (Automation Contributor, Role ID f353d9bd) ja | Logic App (Logic App Contributor, 87a39d53) ja | Event Hub (Event Hubs Data Sender, 2b629674) ja | Azure Function nei | Webhook nei | Secure Webhook nei | ITSM Connector nei\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/alerts/action-groups\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/alerting-strategies-escalation.md#8\",\n \"claim\": \"Event Hub action: cross-tenant support opptil API-versjon 2023-09-01-preview\",\n \"claim_type\": \"version\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/alerts/action-groups\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/alerting-strategies-escalation.md#9\",\n \"claim\": \"Managed Identity-støtte for Action Groups er i preview; Azure Portal legger automatisk til role assignments, PowerShell/CLI/SDK krever manuell rolletildeling\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/alerts/action-groups\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/alerting-strategies-escalation.md#10\",\n \"claim\": \"Dynamic thresholds i metric alerts: ML-baserte terskler som tilpasser seg automatisk\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/alerts/best-practices-alerts\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/alerting-strategies-escalation.md#11\",\n \"claim\": \"Managed identity for log search alert rules er påkrevd hvis query aksesserer Azure Data Explorer (ADX) eller Azure Resource Graph (ARG)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/alerts/best-practices-alerts\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/alerting-strategies-escalation.md#12\",\n \"claim\": \"Activity Log alerts, Service Health alerts og Resource Health alerts er gratis (ingen kostnad per rule/evaluering); first 10 metric alert rules free per subscription\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/azure-monitor/alerts/best-practices-alerts\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-governance/references/monitoring-observability/alerting-strategies-escalation.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-governance/references/monitoring-observability/security-and-audit-logging-ai.md", + "claim_count": 10, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-governance/references/monitoring-observability/security-and-audit-logging-ai.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-governance/references/monitoring-observability/security-and-audit-logging-ai.md`)\n\n[\n {\n \"id\": \"ms-ai-governance/monitoring-observability/security-and-audit-logging-ai.md#1\",\n \"claim\": \"Resource logs er deaktivert som standard for Azure AI-tjenester, med unntak av Microsoft Foundry som har automatisk (auto-enabled) logging til Azure Monitor/Log Analytics\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/diagnostic-logging\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/security-and-audit-logging-ai.md#2\",\n \"claim\": \"Azure OpenAI loggkategorier: Audit, RequestResponse, Trace (krever diagnostic setting, ikke enabled som standard)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/foundry-classic/openai/how-to/monitor-openai\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/security-and-audit-logging-ai.md#3\",\n \"claim\": \"Azure AI Services loggkategorier: Audit, RequestResponse, AllMetrics (krever diagnostic setting)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/ai-services/diagnostic-logging\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/security-and-audit-logging-ai.md#4\",\n \"claim\": \"Microsoft Defender for Cloud AI threat protection + AI-SPM: støtter Azure OpenAI; konfigureres separat for Foundry-ressurser; «Foundry resource» (kind=AIServices) er nytt navn på tidligere «Foundry Tools»\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/security/benchmark/azure/mcsb-v2-artificial-intelligence-security\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/security-and-audit-logging-ai.md#5\",\n \"claim\": \"Log Analytics retention limit: maks 730 dager uten export\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/security-and-audit-logging-ai.md#6\",\n \"claim\": \"Log Analytics retention er gratis de første 31 dagene\",\n \"claim_type\": \"tpm\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/security-and-audit-logging-ai.md#7\",\n \"claim\": \"Azure Policy built-in policy finnes: «Diagnostic logs in Azure AI services resources should be enabled»\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/governance/policy/samples/azure-security-benchmark\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/security-and-audit-logging-ai.md#8\",\n \"claim\": \"Azure security baseline for Azure OpenAI er basert på MCSB v1.0; produktet refereres som «Foundry Tools» i baseline-dokumentet\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/security/benchmark/azure/baselines/azure-openai-security-baseline\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/security-and-audit-logging-ai.md#9\",\n \"claim\": \"Microsoft Foundry security baseline (MCSB v1.0): tjenesten omdøpt til «Microsoft Foundry»; Customer Lockbox ikke støttet; lokal autentisering til data plane ikke støttet; DLP/sensitive data discovery ikke støttet nativt\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/security/benchmark/azure/baselines/azure-ai-foundry-security-baseline\"\n },\n {\n \"id\": \"ms-ai-governance/monitoring-observability/security-and-audit-logging-ai.md#10\",\n \"claim\": \"Databricks Unity Catalog: sentral governance for både data og AI assets (modeller, features, lineage til kolonnenivå); tre governance-modeller (sentralisert, distribuert/federated, hybrid); audit-logging på workspace- og account-nivå + verbose audit logs; AI-genererte kommentarer støttes\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/databricks/lakehouse-architecture/data-governance/best-practices\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-governance/references/monitoring-observability/security-and-audit-logging-ai.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-governance/references/norwegian-public-sector-governance/accessibility-requirements-wcag-norway.md", + "claim_count": 9, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-governance/references/norwegian-public-sector-governance/accessibility-requirements-wcag-norway.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-governance/references/norwegian-public-sector-governance/accessibility-requirements-wcag-norway.md`)\n\n[\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/accessibility-requirements-wcag-norway.md#1\",\n \"claim\": \"Copilot Studio flows finnes i to format: agent flows (eksisterende, Power Automate-lignende) og Workflows (nytt format med revidert visuell designer, public preview); begge kan utløses manuelt, av hendelser/agenter eller tidsplan\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/microsoft-copilot-studio/fundamentals-what-is-copilot-studio#accessibility\"\n },\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/accessibility-requirements-wcag-norway.md#2\",\n \"claim\": \"Copilot Studio bruker Azure OpenAI GPT-modell for AI-kapabiliteter\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/microsoft-copilot-studio/fundamentals-what-is-copilot-studio#accessibility\"\n },\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/accessibility-requirements-wcag-norway.md#3\",\n \"claim\": \"Retirement: etter utgangen av juni 2026 kan ikke Copilot Studio for Teams-appen brukes til å lage klassiske chatbots — makers omdirigeres til Copilot Studio-webappen\",\n \"claim_type\": \"status\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/microsoft-copilot-studio/fundamentals-what-is-copilot-studio#accessibility\"\n },\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/accessibility-requirements-wcag-norway.md#4\",\n \"claim\": \"Copilot Studio innebygd tilgjengelighet: authoring canvas bygget etter Microsoft accessibility guidelines; standard navigasjonsmønstre (tastatur, skjermleser); ARIA-semantikk for adaptive card rendering; flerspråklig støtte inkludert norsk\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/microsoft-copilot-studio/fundamentals-what-is-copilot-studio#accessibility\"\n },\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/accessibility-requirements-wcag-norway.md#5\",\n \"claim\": \"Azure AI Speech norsk språkstøtte: bokmål (nb-NO), nynorsk (nn-NO), Custom Speech for dialektvarianter; funksjoner: real-time transcription, speaker diarization, profanity filter/content moderation, batch-transkripsjon\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/accessibility-requirements-wcag-norway.md#6\",\n \"claim\": \"Azure AI Vision-kapabiliteter: Image Analysis API (beskrivende bildetekster), OCR, Face API (attributtgjenkjenning); innebygd content moderation og PII detection\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/accessibility-requirements-wcag-norway.md#7\",\n \"claim\": \"Produktnavn: Azure AI Document Intelligence (tidligere Form Recognizer)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/accessibility-requirements-wcag-norway.md#8\",\n \"claim\": \"Azure OpenAI Service Content Safety: automatisk filtrering av hatefullt språk/vold/seksuelt innhold, jailbreak detection, groundedness detection\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n },\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/accessibility-requirements-wcag-norway.md#9\",\n \"claim\": \"Power Apps støtter skjermlesere (Narrator, JAWS, NVDA), tastaturnavigasjon uten mus og høykontrast-modus\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": null\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-governance/references/norwegian-public-sector-governance/accessibility-requirements-wcag-norway.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + }, + { + "file": "skills/ms-ai-governance/references/norwegian-public-sector-governance/budget-and-accounting-ai-costs.md", + "claim_count": 6, + "prompt": "# Per-claim groundedness judge — bake-off **v3.1** (recall-hardened over v3's 3 confirmed FNs)\n\nv3.1 of `judge-claim-prompt-v3.md`. Same blind, per-claim, one-subagent-per-file\ndesign, same three verdicts, same output schema, same evidence discipline. v3.1\nchanges only **three reasoning rules** (R1, R7, and a new R8), each fixing one of the\n**3 false negatives v3 still carried** (the judge-vs-gold disagreements G5 confirmed\nwere genuine judge misses, not stale gold).\n\n**Why v3.1 exists — and what it is NOT (transparent, not p-hacking).** The G5b\nfreshness spot-check (2026-06-30) blind-re-adjudicated v3's 4 apparent *false\npositives* against live Microsoft Learn. **All 4 turned out to be stale gold — v3\nflagged every one correctly** (`adr-template#1` \"zero permission management\" is\ncontradicted; `multi-region#2` lists retired `gpt-35-turbo`; `network-resilience#4`\noverstates \"recommended\" as \"obligatorisk\"; `vector-storage#7` cites the wrong GA\ndate). So **v3 has zero real false positives** (P = 100% on corrected gold), and the\nprecision-side \"FP-vakt\" originally planned for v3.1 is **dropped — there is nothing to\ndefend.** v3.1 is therefore a **pure recall hardening**: it tightens three rules so the\njudge catches 3 documented failure modes it currently misses, without touching the\nprecision-side rules (R2, R5, R6 and v3's R7 capability-following are unchanged).\n\n**Adoption is gated on measurement, not assertion — and the bar is now v3 (P 100% /\nR 92.9% on corrected gold), not v2.** v3 sits at the precision ceiling, so v3.1 can only\nbe adopted if it **holds P = 100% AND lifts R above 92.9%** (catches FNs without\nintroducing a single new false positive). Any new FP drops P below 100% and fails the\ngate — keep v3. Recall rules are double-edged over the full population (v3's own\nbake-off taught this), so the 45-way fan-out, not this prose, decides. v3 results stay\nfrozen; v3.1 writes to `judge-bakeoff-results-v3.1.json` and is graded against the\ncorrected `gold-correctness-set.json`.\n\n---\n\nYou are a correctness judge for Microsoft AI reference documentation. You verify\nfactual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).\nBe strict and adversarial — do not give the benefit of the doubt, do not pad, do not\ninfer a value the source does not state.\n\nYou are judging claims extracted from `skills/ms-ai-governance/references/norwegian-public-sector-governance/budget-and-accounting-ai-costs.md`. For EACH claim in the batch below,\ndecide whether the cited Microsoft Learn source **grounds** the claim.\n\n## The three verdicts (exhaustive, mutually exclusive)\n\n- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed\n value(s). The page supports the claim. (Maps to gold `correct`.)\n- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a\n **different / contradicting / superseded** value for what the claim asserts. The\n claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)\n- **`source_silent`** — you fetched the cited page (and searched as a fallback) but\n **no** `learn.microsoft.com` page states the claimed value at all. You cannot\n confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure\n pages typically lands here — that is expected, not a failure. **Exception: existence\n claims — see Rule R2.**\n\n## ⚠️ EXACT-VALUE RULE (inherited from v2 — still in force)\n\nA claim is `grounded` ONLY if the fetched page states the **exact** asserted value(s).\nVerifying that the page \"is about\" the SKU/model/feature is **not** enough — the\nspecific number, name, date, tier, dimension, or status must match. If the claim\nasserts value **X** and the page states a **different** value **Y** (even if adjacent\nor plausible), the verdict is **`not_grounded`**. This rule does NOT lower the bar for\n`not_grounded`: you still need a fetched quote stating the **differing** value.\n\nApplies with special force to `sku`, `taxonomy`, `version`, `tpm`, `region`, `status`.\n\n---\n\n## CALIBRATION RULES — read all eight before judging\n\nThe exact-value rule is a blunt instrument. The 8 rules below sharpen it on both\nedges: **R1–R4 and R8 catch real errors** (more `not_grounded`); **R5–R7 stop\nover-flagging where the core is grounded** (more correctly `grounded`). When a rule\nbelow conflicts with a literal reading of the exact-value rule, the rule below governs\n— it is the more precise standard.\n\n### Recall side — flag these as `not_grounded`\n\n**R1 — Bound understatement / overstatement (fixes FN2; v3.1 splits lower vs upper).**\nA claim may assert a **bound**. Direction matters — judge it by which side the bound\nconstrains:\n\n- **Lower bound** (\"100+\", \"200k+\", \"at least N\", \"minimum N\"): do not auto-`grounded`\n it just because the true value satisfies the inequality. Apply the **lower-bound\n policy:** if the true current value **grossly exceeds** the stated bound — roughly\n **>2× and decision-changing** — the bound materially misleads → `not_grounded`. A\n *tight* lower bound (true value within the same order of magnitude) stays `grounded`.\n *Example (FN2): \"200k+ context\" while the page states 1,047,576 (~1M) — ~5× → `not_grounded`.*\n- **Upper bound** (\"up to N\", \"opptil N\", \"maximum N\", \"no more than N\", \"as many as N\"):\n this is a **ceiling**, not a floor. The lower-bound leniency does **NOT** apply. The\n exact-value rule governs: if the live page states a current maximum **higher** than N,\n the stated ceiling is **superseded** → `not_grounded` — **regardless of ratio** (even\n a 1.1× exceedance breaks a ceiling). The claim tells the reader the limit is N when it\n is really higher. *Example (v3-FN): claim \"up to 18 underlying models\" while the page\n states 28 → the ceiling has moved → `not_grounded`.* (Only `grounded` if the true\n maximum is N or the claim's ceiling still binds.)\n\n**R2 — `source_silent` does NOT excuse an existence claim (fixes FN3, FN5).** When the\nclaim asserts that a named entity **exists / is offered / is in a list** (\"X is a\nbuilt-in judge\", \"feature Y is available\", \"tier Z exists\"), and you fetch the\nauthoritative page that *would* enumerate it and the entity is **absent**, that absence\nis **evidence the claim is wrong** — return `not_grounded`, not `source_silent`. Reserve\n`source_silent` for values a page would not be expected to enumerate (e.g. JS-rendered\nprices). State in `reason` that you checked the canonical enumerating page and the\nentity was not present. *Example (FN5): claim \"99.99% SLA tier\" while the reliability\npage lists only 99.9% → absence of any 99.99% tier = `not_grounded`.*\n\n**R3 — Frame/unit replacement (fixes FN4).** A claim's **organizing frame or unit** can\nbe superseded even when derived ratios survive. If the page shows the claim's framing\nhas been **replaced** (e.g. \"1 Unit Capacity\" → \"Quota Tiers\"; a renamed/retired\nmetric), the claim is `not_grounded` even if some embedded numbers still appear\nsomewhere — the claim describes a world that no longer exists. Check that the *unit and\nstructure* the claim assumes still match the current page, not just the digits.\n\n**R4 — Current row, never a legacy row (fixes FN6).** Pages often carry historical or\neffective-dated rows (\"Before April 3, 2024\", \"Legacy\", \"Retiring\"). A claim is\n`grounded` only if it matches the **current/effective** row. Matching a clearly\ntime-stamped *past* row is `not_grounded` (the value has since changed). Always locate\nthe row that applies *today*. *Example (FN6): storage limits matching only the\n\"Before April 3, 2024\" row while current limits differ → `not_grounded`.*\n\n**R8 — Multi-part claims: every load-bearing part must hold (v3.1 — fixes the\n`ai-foundry-dr#9` FN).** A single claim often bundles **several load-bearing\nsub-assertions** (a status AND a region; a capability AND a named target; a date AND a\nGA level). Verify **each load-bearing part separately**. If **any one** load-bearing\npart is contradicted by the source, the whole claim is `not_grounded` — even when the\nother parts check out. Do not let a correct first half earn a `grounded` for a wrong\nsecond half. *Example (v3-FN): \"Global training (Public Preview), cheaper, no data\nresidency; use regional in Norway East\" — the GA-vs-Preview part and the \"no residency\"\npart hold, but **Norway East is a Global (non-residency) training region, not a regional\none** → one load-bearing part is wrong → `not_grounded`.* (R8 is the mirror of R6:\nR6 forgives an **omitted, non-load-bearing** detail; R8 condemns a **stated,\nload-bearing** part that is wrong. Decide first whether the part is load-bearing — if\nthe claim *asserts* it and a reader would act on it, it is.)\n\n### Precision side — keep these `grounded` (do not over-flag)\n\n**R5 — Documented theoretical↔benchmark equivalence (fixes FP3).** Do not flag a\nnumeric claim merely because the exact string is not verbatim, when the asserted value\nis the **documented theoretical or benchmark equivalent** of what the page states and\nboth trace to Microsoft sources (e.g. a theoretical max vs a measured benchmark of the\nsame technique, same order of magnitude, same direction). The exact-value rule targets\n*drifted/contradicting* values — not two Microsoft-sourced expressions of the same\nfact. If the page substantiates the magnitude and the technique, keep `grounded` and\nnote the equivalence in `reason`.\n\n**R6 — Core grounded, detail omitted ≠ ungrounded (fixes FP4).** Distinguish \"the\nclaim's **core** assertion is grounded but it omits a sub-category\" from \"the core is\nungrounded.\" If the page confirms the claim's **central** behavior/categorization and\nthe only gap is an *unstated additional* case the claim did not deny, that is\n`grounded` (the claim is incomplete, not wrong). Reserve `not_grounded` for when the\npage **maps the core differently** or the claim **asserts** something the page\ncontradicts. Omission ≠ contradiction. (Contrast R8: an omitted case is forgiven here;\na *stated* but wrong load-bearing part is not — that is R8's domain.)\n\n**R7 — Follow the capability to its canonical page; don't punish illustrative numbers\n(fixes FP5; v3.1 sharpens the load-bearing carve-out).** If a claim asserts a **real\ncapability** and the cited `evidence_url` does not foreground it, search for the\n**canonical** page that documents the capability before judging — do not return\n`not_grounded` merely because the *cited* page was a weak choice. And when a capability\nis solidly grounded, do **not** flag it over an *illustrative* attached number (e.g.\n\"~0 RTO/RPO\", \"≈15 min\") that the claim offers as an order-of-magnitude illustration\nrather than a cited spec. Judge the **capability**; treat an illustrative figure as\ngrounded if the capability is.\n\n> **⚠️ Load-bearing carve-out (v3.1, fixes the `token-usage#3` FN).** R7's leniency\n> covers only *illustrative* values. It does **NOT** cover a value or **exact string**\n> that **IS the assertion** — a metric name, an API field, an SDK identifier, an enum\n> value, a specific date/version. When the claim's load-bearing content is the literal\n> name/string itself (e.g. \"the metrics are `PromptTokens` and `CompletionTokens`\"),\n> the exact-value rule applies in full: if the live page names them differently\n> (`ProcessedPromptTokens` / `InputTokens` / `GeneratedTokens` / `OutputTokens`), the\n> claim is `not_grounded`. \"Follow to the canonical page\" means find the **right\n> names**, not rescue wrong ones. A reader would copy that string into code; an\n> illustrative magnitude they would not.\n\n---\n\n## Procedure (per claim)\n\n1. **Identify the volatile assertion(s)** in the claim text — and when the claim\n bundles several (R8), enumerate **each load-bearing part**. The `claim_type` tells\n you what to check:\n - `version` → model/API version, GA date, context window, max output, training cutoff\n - `tpm` → tokens-per-minute / throughput / quota numbers\n - `sku` → SKU name, tier, PTU minimums, deployment type\n - `region` → regional availability\n - `status` → GA / preview / retirement / deprecation status\n - `taxonomy` → categorization, capability mapping, which-feature-does-what\n2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's\n `evidence_url`. If the claim has no `evidence_url`, or the fetched page does not\n address the assertion, run `microsoft_docs_search` to find the authoritative page.\n **Under R2/R7, actively seek the canonical enumerating/capability page** — a weak\n cited URL is not the last word.\n3. **Exact-value entailment check** each checkable value (and each load-bearing part\n under R8), then apply the calibration rules R1–R8. Classify which rule(s), if any,\n govern the claim. For R1, first decide whether the bound is a **lower** bound (floor)\n or an **upper** bound (ceiling) — they invert.\n4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a verbatim\n quote you actually fetched from a `learn.microsoft.com` URL. For R2 (existence\n absence), the quote is the canonical enumeration in which the entity does **not**\n appear — quote the enumeration and state the entity is absent. No quote → `source_silent`.\n\n## Hard rules\n\n- Verify against the fetched page only. Do not rely on prior knowledge of model\n specs / prices — those are exactly what may have drifted.\n- Stable identifiers are not volatile and are not your job to refute: regulation year\n (2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10\n 2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on\n whatever volatile value it carries, else `source_silent`.\n- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.\n- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the\n verdict (empty string for `source_silent`). `evidence_url` = the page you actually\n used (may differ from the cited one if you fell back to search).\n- `rule` = which calibration rule governed, if any (`R1`–`R8`), else empty.\n\n## Batch to judge (from `skills/ms-ai-governance/references/norwegian-public-sector-governance/budget-and-accounting-ai-costs.md`)\n\n[\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/budget-and-accounting-ai-costs.md#1\",\n \"claim\": \"Azure Cost Management-verktøy: Cost Analysis, Budgets, Cost Alerts, Advisor Recommendations\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/cloud-adoption-framework/ready/azure-setup-guide/manage-costs\"\n },\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/budget-and-accounting-ai-costs.md#2\",\n \"claim\": \"Budsjetter kan opprettes på subscription-nivå, resource group-nivå og tag-basert\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/cost-management-billing/costs/tutorial-acm-create-budgets\"\n },\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/budget-and-accounting-ai-costs.md#3\",\n \"claim\": \"Budgettyper: Actual cost budget (faktisk forbruk) og Forecasted cost budget (forventet forbruk)\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/cost-management-billing/costs/tutorial-acm-create-budgets\"\n },\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/budget-and-accounting-ai-costs.md#4\",\n \"claim\": \"Budget alerts: e-postvarsel ved 50 %, 80 %, 100 % av budsjettgrensen; integrasjon med Azure Logic Apps for automatiske tiltak\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/cost-management-billing/costs/tutorial-acm-create-budgets\"\n },\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/budget-and-accounting-ai-costs.md#5\",\n \"claim\": \"Azure Cost Management støtter tag inheritance og cost allocation rules for automatisk tagging av underliggende ressurser\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/cost-management-billing/costs/overview-cost-management\"\n },\n {\n \"id\": \"ms-ai-governance/norwegian-public-sector-governance/budget-and-accounting-ai-costs.md#6\",\n \"claim\": \"Kostnadsdata kan eksporteres automatisk til Azure Storage (daglig/ukentlig/månedlig), med detaljert rapportering per ressurs, tag og tidsperiode\",\n \"claim_type\": \"taxonomy\",\n \"evidence_url\": \"https://learn.microsoft.com/en-us/azure/cost-management-billing/costs/tutorial-improved-exports\"\n }\n]\n\n## Output (strict JSON, no fence)\n\n```\n{\"file\":\"skills/ms-ai-governance/references/norwegian-public-sector-governance/budget-and-accounting-ai-costs.md\",\"results\":[\n {\"id\":\"\",\"judge_verdict\":\"grounded|not_grounded|source_silent\",\"rule\":\"\",\"evidence_url\":\"\",\"evidence_quote\":\"\",\"reason\":\"\"}\n]}\n```\n" + } +]