diff --git a/scripts/kb-eval/data/judge-results.json b/scripts/kb-eval/data/judge-results.json index 6c0e6c4..700acbb 100644 --- a/scripts/kb-eval/data/judge-results.json +++ b/scripts/kb-eval/data/judge-results.json @@ -20,11 +20,12 @@ "K9_noTimeSensitive": { "pass": false, "findings": ["Line 119: 'Foundry Agent Service GA' — explicit GA-status claim in body", "GPT-4o/Whisper/text-embedding-3/Florence — version-pinned product names in body", "Line 48: 'Agent Framework (erstatter Semantic Kernel Agents)' — lifecycle/transition claim", "A2A/CUA/Foundry Workflows — era-bound feature names"] } }, "ms-ai-governance": { - "K1_triggerPrecision": { "provisional": true, "precision": 0.95, "notes": "19/20. Miss: Schrems II / data-transfer — body section 2.3 covers it but description has NO Schrems/dataoverføring keyword. Add trigger phrase to description." }, - "K4_noDuplication": { "score": 3, "pass": false, "evidence": "Moderate duplication: §6.1 DPIA risk-factor tree (incl. >=2-faktorer threshold) duplicates dpia-norwegian-methodology-ai.md; §6.2 + §2.1 AI Act taxonomy duplicates ai-act-classification-methodology.md; §1.2 Digdir 7-principle table restates per-principle files. Regulatory update must be applied twice." }, - "K7_imperativeStyle": { "ratio": 1.0, "pass": true, "notes": "10/10 imperative." }, - "K8_sourceCitation": { "ratio": 1.0, "pass": true, "notes": "5/5 dated headers + Status + inline source citations (Lovdata/NSM/europalov). BONUS DEFECT: SKILL.md line 191 references 'drift-detection-automated-retraining.md' which does NOT exist (actual: model-performance-drift-detection.md) — broken ref path." }, - "K9_noTimeSensitive": { "pass": false, "findings": ["Line 121: 'Microsoft EU Data Boundary ... Azure OpenAI (Sweden Central, West Europe)' — volatile region/availability claim in body (most genuine finding)", "Lines 81/121: 'Regulation 2024/1689' + 'Schrems II (C-311/18)' — legal identifiers (stable, borderline)", "§2.1/§6.2 AI Act obligation status presented without Digital Omnibus caveat — may date"] } + "_updated": "S9 (2026-06-20) — K4+K9 body-cleanup re-judge", + "K1_triggerPrecision": { "provisional": true, "precision": 0.85, "notes": "17/20 (7/10 in-domain hits, 10/10 out-of-domain correct). RECALL GAP: description has no Schrems II/dataoverføring/TIA trigger though §2.3 covers it; GDPR + 'consult Datatilsynet' also under-represented vs body scope. Operator must curate + add Schrems II trigger (S11)." }, + "K4_noDuplication": { "score": 5, "pass": true, "evidence": "S9 FIX: §6.2 now a compact decision-flow with explicit pointers ('[full forbudsliste i ai-act-classification-methodology.md]', '[åtte kategorier ...]') — Art.5 + Annex III lists no longer enumerated in body; they live only in references/responsible-ai/ai-act-classification-methodology.md. §2.1 is the single 4-level overview table ('ikke gjenta dem her'). §6.1 (DPIA tree), §1.2 (Digdir table), §6.3/§6.4 are routing/orientation, not verbatim copies of ref files." }, + "K7_imperativeStyle": { "ratio": 1.0, "pass": true, "notes": "10/10 sampled instruction sentences imperative/infinitive." }, + "K8_sourceCitation": { "ratio": 1.0, "pass": true, "notes": "5/5 sampled refs carry Last updated + Status + Category headers." }, + "K9_noTimeSensitive": { "pass": true, "findings": [] } }, "ms-ai-infrastructure": { "K1_triggerPrecision": { "provisional": true, "precision": 1.0, "notes": "20/20. Tightly scoped + named trigger phrases, low false-positive risk. Borderline (hybrid RAG, multi-region cost) need real-corpus validation. Operator must curate." }, @@ -34,10 +35,11 @@ "K9_noTimeSensitive": { "pass": false, "findings": ["Lines 93-98: SLA table w/ hardcoded percentages (99.9/99.999/99.95%) + 'Standard v2'", "Lines 149/158/184/186/249: Phi-3/Phi-4 model versions + param counts (3.8B/14B) in body", "Line 141: 'Azure Local (tidl. Azure Stack HCI)' rename note", "Line 87: hardcoded 'peak + 30% buffer'"] } }, "ms-ai-security": { - "K1_triggerPrecision": { "provisional": true, "precision": 0.95, "notes": "19/20. Clean sibling separation (AI Act/DPIA/platform/RAG/BCDR not triggered). One ambiguity: model feature-comparison could be pulled by broad 'performance optimization for AI'. Operator must curate." }, - "K4_noDuplication": { "score": 3, "pass": false, "evidence": "DUPLICATION-WITH-CONTRADICTION: body weighting table (L52-59 Standard: Identity 20/Network 15/Data 20/Content 20/Compliance 15/Monitoring 10) CONTRADICTS canonical rubric security-scoring-rubrics-6x5.md (Compliance 25/Data 20/Identity 20/Content 15/Network 10/Monitoring 10). Body scoring rule (weighted sum) also diverges from rubric (Ja-checkpoint count). Real correctness bug." }, - "K7_imperativeStyle": { "ratio": 1.0, "pass": true, "notes": "10/10 imperative." }, - "K8_sourceCitation": { "ratio": 1.0, "pass": true, "notes": "5/5 dated headers; mostly no header source-URL (URLs in body cost-register rows)." }, - "K9_noTimeSensitive": { "pass": false, "findings": ["Line 73: 'OWASP LLM Top 10 (2025)' — dated standard version", "Line 92: 'Defender ... GA for AI applications, Preview for AI agents' + 'ikke i Azure Government' — GA/preview + region status in body", "Line 138: 'GPT-4o mini vs GPT-4o' — model versions", "Lines 149-153: hardcoded perf/price figures (20-50ms, 5-10x, 50%/80%, Batch 50% @ 24h SLA)"] } + "_updated": "S9 (2026-06-20) — K4+K9 body-cleanup; K4 re-judged after S8 weighting fix + S9 risk-table/P10P50P90 fixes", + "K1_triggerPrecision": { "provisional": true, "precision": 1.0, "notes": "20/20 from description; clean sibling separation (governance/engineering/advisor/infrastructure/license not triggered). Borderline: generic latency/perf prompts overlap engineering, but description explicitly claims 'performance optimization for AI'. Operator must curate final 20." }, + "K4_noDuplication": { "score": 5, "pass": true, "evidence": "S9 re-judge (cold, post-fix): (a) 6x5 weights — body L50 routes to security-scoring-rubrics-6x5.md ('Ikke dupliser vekttallene her'), no body numbers; (b) risk-classification thresholds — body L54 routes to same rubric ('Ikke dupliser terskeltallene her'), canonical mapping incl. 1.00-1.49 Uakseptabel lives only in rubric; (c) P10/P50/P90 — body L94 affirms 'per komponent (ikke flat multiplikator)' owned by deterministic-cost-calculation-model.md §3, concrete factors only in cost model — body affirms, does not contradict; (d) OWASP table + §3 perf are routing/orientation with explicit volatile-numbers-live-in-refs note. No duplication/contradiction." }, + "K7_imperativeStyle": { "ratio": 1.0, "pass": true, "notes": "10/10 sampled instruction sentences imperative/infinitive." }, + "K8_sourceCitation": { "ratio": 1.0, "pass": true, "notes": "5/5 dated headers + Status; 3/5 also carry Verified: MCP ." }, + "K9_noTimeSensitive": { "pass": true, "findings": [] } } } diff --git a/scripts/kb-eval/judge-prompt.md b/scripts/kb-eval/judge-prompt.md index 341ddd9..9b5ee73 100644 --- a/scripts/kb-eval/judge-prompt.md +++ b/scripts/kb-eval/judge-prompt.md @@ -24,8 +24,12 @@ Vær streng og adversariell — ikke ros, ikke pynt på tall. imperativ/infinitiv. pass = ratio ≥ 0.80. - **K8 kildehenvisning i ref-filer:** for 5 samplede ref-filer, har header `Last updated` / `Verified` / kilde-URL? Rapporter andel. pass = ratio ≥ 0.80. -- **K9 ingen tid-sensitiv info i SKILL.md body:** finnes datoer/versjoner/GA/preview-status DIREKTE - i body (ikke i ref-filer)? List funn. pass = ingen funn. +- **K9 ingen VOLATIL tid-sensitiv info i SKILL.md body:** finnes **volatile** påstander DIREKTE i body + (ikke i ref-filer) — GA/preview-release-status, modellversjoner, SLA-/ytelses-/pris-tall, regions- + tilgjengelighet? List funn. pass = ingen volatile funn. **Utenfor scope (ikke funn):** stabile + identifikatorer som forordningsår (2024/1689), lovsaksnr (C-311/18), standard-versjonsnavn + (OWASP LLM Top 10 2025, MADR v3.0), filnavn og generisk forklarende «preview»/«GA» uten konkret + produkt-status. Disse er navngitte identifikatorer, ikke ferskhets-sensitive påstander. RETURNER KUN dette JSON-objektet (ingen annen tekst, ingen markdown-fence): diff --git a/skills/ms-ai-governance/SKILL.md b/skills/ms-ai-governance/SKILL.md index c57b796..91820ae 100644 --- a/skills/ms-ai-governance/SKILL.md +++ b/skills/ms-ai-governance/SKILL.md @@ -118,8 +118,9 @@ High-risk systems require: risk management system, data governance, technical do ### 2.3 Schrems II og dataoverfoering -Schrems II (C-311/18) requires Transfer Impact Assessment for AI systems using US cloud providers. For Azure/Microsoft: map data flows, use SCCs as primary transfer basis, assess FISA 702/CLOUD Act exposure, implement supplementary measures (encryption, pseudonymization). Microsoft EU Data Boundary ensures processing within EU/EEA for core services including Azure OpenAI Service (Sweden Central, West Europe). +Schrems II (C-311/18) requires Transfer Impact Assessment for AI systems using US cloud providers. For Azure/Microsoft: map data flows, use SCCs as primary transfer basis, assess FISA 702/CLOUD Act exposure, implement supplementary measures (encryption, pseudonymization). Microsoft EU Data Boundary keeps processing within EU/EEA for core services including Azure OpenAI Service; gjeldende regioner og dekningsomfang er volatile — verifiser mot ref-fila før du oppgir dem. +> **Reference:** `references/responsible-ai/gdpr-compliance-ai-systems.md` (EU Data Boundary, dataoverføringsgrunnlag) > **Cross-reference:** `skills/ms-ai-security/references/ai-security-engineering/data-leakage-prevention-ai.md` --- @@ -239,32 +240,21 @@ Resultat: ### 6.2 AI Act risikoklassifisering ``` -Er AI-systemet paa forbudslisten (Art. 5)? -├── Sosial scoring av myndigheter -├── Utnyttelse av saarbare grupper -├── Biometrisk fjernidentifisering i sanntid (unntak: alvorlig kriminalitet) -├── Emotion recognition paa arbeidsplass/skole (unntak: sikkerhet/medisin) -└── Ja til noen → UAKSEPTABEL RISIKO — Forbudt +Forbudt praksis (Art. 5)? [full forbudsliste i ai-act-classification-methodology.md] +└── Ja → UAKSEPTABEL RISIKO — Forbudt -Er AI-systemet i Annex III? -├── Biometrisk identifisering -├── Kritisk infrastruktur -├── Utdanning/opplaering -├── Ansettelse/personal -├── Essensielle offentlige tjenester -├── Rettshåndhevelse -├── Migrasjon/grensekontroll -├── Rettsforvaltning/demokrati -└── Ja til noen → HOEY RISIKO — Full compliance-krav +I Annex III? [åtte kategorier i ai-act-classification-methodology.md + ai-act-annex-iii-checklist.md] +└── Ja → HOEY RISIKO — Full compliance-krav -Samhandler systemet direkte med borgere? -├── Chatbot, innholdsgenerering, deepfakes +Samhandler systemet direkte med borgere (chatbot, innholdsgenerering, deepfakes)? └── Ja → BEGRENSET RISIKO — Transparenskrav Ingen av de ovennevnte? └── MINIMAL RISIKO — Frivillig Code of Conduct ``` +Full metodikk med eksempler og grensetilfeller: kjør `/architect:classify`, eller se `references/responsible-ai/ai-act-classification-methodology.md`. Risikonivåene og kravene per nivå står i §2.1 — ikke gjenta dem her. + ### 6.3 Naar skal Datatilsynet konsulteres? ``` diff --git a/skills/ms-ai-security/SKILL.md b/skills/ms-ai-security/SKILL.md index eed5acf..5d11a6b 100644 --- a/skills/ms-ai-security/SKILL.md +++ b/skills/ms-ai-security/SKILL.md @@ -51,13 +51,7 @@ Apply weights based on workload type, then calculate: **Samlet score = Sum(dimen ### Risikoklassifisering -| Samlet score | Klassifisering | Anbefaling | -|-------------|---------------|------------| -| 1.0 - 2.0 | Kritisk risiko | Stopp utrulling. Umiddelbar utbedring. | -| 2.1 - 3.0 | Høy risiko | Begrenset tilgang. Utbedringsplan innen 30 dager. | -| 3.1 - 3.5 | Moderat risiko | Produksjon med restriksjoner. Utbedringsplan innen 90 dager. | -| 3.6 - 4.5 | Lav risiko | Produksjon godkjent. Kontinuerlig forbedring. | -| 4.6 - 5.0 | Minimal risiko | Produksjon godkjent. Benchmark for andre løsninger. | +Etter at samlet score er beregnet, klassifiser løsningen i risikokategori med anbefalt handling. De kanoniske terskelverdiene (samlet score → risikokategori, inkludert «Uakseptabel»-kategorien) ligger i samme rubrikkfil som vektene — `references/ai-security-engineering/security-scoring-rubrics-6x5.md` — slik at klassifiseringen holdes konsistent med `security-assessment-agent`. Ikke dupliser terskeltallene her. For fullstendige rubrikker med eksempler per dimensjon og score, see `references/ai-security-engineering/security-scoring-rubrics-6x5.md` and `references/ai-security-engineering/ai-security-scoring-framework.md`. @@ -80,7 +74,7 @@ Map each threat to the solution under assessment. Use the reference files for de All reference files are in `references/ai-security-engineering/`. LLM04/06/08/09 deler den konsoliderte filen `references/ai-security-engineering/owasp-llm-top10-azure-mitigations.md`; LLM10 dekkes av rate-limit-/kostnadsfiler. -Kjøretids-trusseldeteksjon for AI-endepunkter dekkes av `defender-threat-protection-ai-services.md` (Defender for Cloud AI threat protection — GA for AI applications, Preview for AI agents; merk: **ikke** tilgjengelig i Azure Government). +Kjøretids-trusseldeteksjon for AI-endepunkter dekkes av `references/ai-security-engineering/defender-threat-protection-ai-services.md` (Defender for Cloud AI threat protection). Release-status, agent-dekning og regionstilgjengelighet — inkludert begrensningen for Azure Government — er dokumentert i ref-fila; verifiser mot den før du oppgir status. ### Azure AI-spesifikke sikkerhetskontroller @@ -97,15 +91,9 @@ For detailed per-service security controls tables, see `references/ai-security-e ### P10/P50/P90 konfidensintervaller -Provide all estimates with three scenarios. Verify current prices via `microsoft_docs_search` before calculating. +Provide all estimates with three scenarios — P10 (lavt volum / minimumskostnad), P50 (forventet/median), P90 (høyt volum / worst-case budsjettering). De kanoniske usikkerhetsfaktorene beregnes **per komponent** (ikke som én flat multiplikator) og eies av `references/cost-optimization/deterministic-cost-calculation-model.md` §3 — bruk faktorene derfra, slik at estimatet er konsistent med `cost-estimation-agent` (som har modellen som OBLIGATORISK kilde). Verify current prices via `microsoft_docs_search` before calculating. -| Scenario | Persentil | Beskrivelse | Multiplikator | -|----------|-----------|-------------|---------------| -| **P10** (Optimistisk) | 10. | Lavt volum, ideelle forhold | Basis x 0.6 | -| **P50** (Forventet) | 50. | Normal bruk, erfaringstall | Basis x 1.0 | -| **P90** (Konservativt) | 90. | Høyt volum, buffer for uforutsett | Basis x 1.8 | - -Adjust multipliers based on historical volatility. Always present both USD and NOK (add 3-5% currency buffer for NOK). +Always present both USD and NOK (add 3-5% currency buffer for NOK). ### TCO-komponenter @@ -126,8 +114,8 @@ See `references/cost-optimization/deterministic-cost-calculation-model.md` and ` Apply these optimization strategies and refer to detailed guidance in references: -- **Token-optimalisering:** Shorter prompts, context window management, model tiering (GPT-4o mini vs GPT-4o), prompt caching. See `references/cost-optimization/token-counting-optimization.md`. -- **PTU vs Pay-As-You-Go:** PTU for stable workloads (break-even ~60-70% utilization), PAYG for variable. See `references/cost-optimization/ptu-vs-paygo-economics.md`. +- **Token-optimalisering:** Shorter prompts, context window management, model tiering (bruk en rimeligere modell for enkle oppgaver), prompt caching. See `references/cost-optimization/token-counting-optimization.md` and `references/cost-optimization/model-selection-price-performance.md` for gjeldende modellvalg. +- **PTU vs Pay-As-You-Go:** PTU for stable, høyt-volum-arbeidsbelastninger (over et break-even-punkt for utnyttelse), PAYG for variable. Konkret break-even-terskel er pris-avhengig og ligger i ref-fila. See `references/cost-optimization/ptu-vs-paygo-economics.md`. - **Caching:** Semantic caching, prompt caching, RAG result caching. See `references/cost-optimization/semantic-caching-patterns.md`. - **Right-sizing:** Start with lowest SKU, monitor 2-4 weeks, consider SLMs for specialized tasks. See `references/cost-optimization/model-selection-price-performance.md`. @@ -135,12 +123,12 @@ Apply these optimization strategies and refer to detailed guidance in references ## 3. Ytelse og skalerbarhet -Optimize latency, throughput, and scalability for AI workloads. Key strategies: +Optimize latency, throughput, and scalability for AI workloads. Konkrete tall — latens-reduksjon, caching-rabatt og batch-prising — er volatile og ligger i ref-filene; oppgi dem aldri fra hukommelsen. Key strategies: -- **Regional deployment** in Norway East / West Europe reduces latency 20-50ms -- **Streaming responses** reduce perceived latency 5-10x for interactive use -- **Prompt caching** gives up to 50% cost reduction and 80% latency reduction for repeated prefixes (>1024 tokens) -- **Batch API** provides 50% price reduction for non-interactive workloads (24h SLA) +- **Regional deployment** in Norway East / West Europe to reduce latency. See `references/performance-scalability/latency-optimization-azure-openai.md`. +- **Streaming responses** to reduce perceived latency for interactive use. See `references/performance-scalability/streaming-response-patterns.md`. +- **Prompt caching** for repeated prompt prefixes to cut both cost and latency. See `references/performance-scalability/prompt-caching-performance.md`. +- **Batch API** for non-interactive workloads at reduced price. See `references/cost-optimization/request-batching-aggregation.md`. - **Auto-scaling patterns:** Horizontal scaling (App Service/AKS), load balancing (APIM/Traffic Manager), queue-based buffering (Service Bus+Functions), PTU+PAYG hybrid - **Rate limit management:** TPM/RPM quotas, exponential backoff with jitter, multi-deployment, APIM for centralized throttling - **Load testing:** Establish baseline, simulate peak traffic, identify breaking points, long-running soak tests