From c778cf6cd25129416b5fc8ac0044ea90a218e777 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Sat, 20 Jun 2026 10:11:45 +0200 Subject: [PATCH] =?UTF-8?q?fix(ms-ai-architect):=20Sesjon=2012=20=E2=80=94?= =?UTF-8?q?=20lukk=20SPOR=20A,=20ny=20eval-baseline=20(K1=E2=80=93K9=20gr?= =?UTF-8?q?=C3=B8nn=20alle=205)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regenerert eval-baseline.json via gated `eval.mjs --write` så baselinen fanger S9+S10+S11 judge-verdiktene (var pre-S9). Eneste endrede artefakt: baseline-fila (123+/128−, ren regenerering — judge-felt null→populert fra data/judge-results.json). Verifisert FØR skriving (read-only) og i resultatet: - judge-cache: K1/K4/K7/K8/K9 PASS alle 5 (advisor K8=0.8 at-threshold, pass) - deterministisk: K2/K3/K5/K6/refTall PASS alle 5 - ny baseline: 0 åpen FAIL K1–K9 på alle 5 skills, judge populert alle 5 SPOR A (krav 1–4) er nå komplett: K1 autoritativ+blindet (S11), 6×5 kanonisk (S8), K4+K9 body-cleanup (S9 sec+gov, S10 eng+infra), skill-optimalisering refTall+K5 (S7). eval-baseline.json er sannhetskilden og grønn på K1–K9. Ingen regresjon: validate 239 · kb-update 122 · kb-eval 15 · kb-integrity 192/192. NESTE: Spor B (scheduled deteksjon · kurs-spor #25 · low-currency · onboarding #26). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01REiKFhP4w6xGXXqWKpPCJJ --- scripts/kb-eval/data/eval-baseline.json | 251 ++++++++++++------------ 1 file changed, 123 insertions(+), 128 deletions(-) diff --git a/scripts/kb-eval/data/eval-baseline.json b/scripts/kb-eval/data/eval-baseline.json index 9b92ff8..eebe433 100644 --- a/scripts/kb-eval/data/eval-baseline.json +++ b/scripts/kb-eval/data/eval-baseline.json @@ -15,27 +15,31 @@ }, "deterministic": { "K2_descriptionFormat": { - "quotedPhrases": 5, + "quotedPhrases": 6, "useWhenForm": true, "imperativeStart": false, "pass": true }, "K3_bodyLength": { - "bodyLines": 240, + "bodyLines": 245, "pass": true }, "K5_progressiveDisclosure": { - "namedFileLinks": 1, + "namedFileLinks": 25, "folderRefs": 5, "totalRefFiles": 62, - "namedRatio": 0.0161, - "pass": false, + "namedRatio": 0.4032, + "pass": true, "sampleNamed": [ - "references/architecture/diagram-prompt-templates.md" + "references/architecture/diagram-prompt-templates.md", + "references/platforms/azure-ai-foundry.md", + "references/platforms/m365-copilot.md", + "references/platforms/copilot-studio.md", + "references/platforms/power-platform.md" ] }, "K6_routingTable": { - "namedStartFiles": 1, + "namedStartFiles": 25, "pass": true }, "refCountConsistency": { @@ -43,18 +47,19 @@ "mismatches": [] }, "K9_timeSensitiveHints": { - "timeSensitiveTokenHits": 7, + "timeSensitiveTokenHits": 8, "sample": [ "Preview", "GA", "preview", + "2026", "v3.0" ] } }, "judgeInputs": { - "description": "Microsoft AI architecture guidance, choosing between Azure AI platforms, Copilot vs Foundry trade-offs. Cosmo Skyberg persona guides through structured problem understanding before technology selection. Specialist in Azure AI Foundry, M365 Copilot, Copilot Studio, Power Platform, Azure OpenAI, Microsoft Agent Framework. Triggers on: \"Microsoft AI architecture\", \"Copilot vs Foundry\", \"which Microsoft AI platform\", \"Cosmo\", \"/architect\".", - "bodyLines": 240, + "description": "Microsoft AI platform selection — choosing between Azure AI Foundry, M365 Copilot, Copilot Studio, Power Platform, Azure OpenAI, and Microsoft Agent Framework for a given scenario. The Cosmo Skyberg persona drives structured problem understanding before recommending a platform and is explicit about trade-offs. Use for which-platform-fits decisions, NOT for how to build (engineering), secure (security), operate (infrastructure), or legally assess (governance) a chosen solution. Triggers on: \"which Microsoft AI platform\", \"Copilot vs Foundry\", \"Copilot Studio or Azure AI Foundry\", \"help me choose an AI platform\", \"Cosmo\", \"/architect\".", + "bodyLines": 245, "refFileSample": [ "skills/ms-ai-advisor/references/architecture/adr-template.md", "skills/ms-ai-advisor/references/architecture/ai-utredning-template.md", @@ -67,10 +72,15 @@ ] }, "judge": { + "_updated": "S11 (2026-06-20) — K1 authoritative (blinded judge vs curated 20-set); description tightened against over-trigging", "K1_triggerPrecision": { - "provisional": true, + "provisional": false, + "inDomainHitRate": 1, + "outDomainFalsePositiveRate": 0, "precision": 1, - "notes": "20/20 from description. BORDERLINE: cost/diagram/DPIA/AI-Act prompts are MS-AI-adjacent; broad 'Microsoft AI architecture' phrase risks over-triggering vs sibling commands. Operator must curate + stress-test sibling overlap." + "pass": true, + "misclassified": [], + "notes": "S11 authoritative: blinded judge vs operator-curated 20-set. 10/10 in-domain hits, 0/10 false positives. Over-trigging fix VALIDATED — judge cited the new 'NOT for build/secure/operate/legally-assess' exclusion when rejecting all four sibling-domain prompts (engineering/security/governance/infra)." }, "K4_noDuplication": { "score": 5, @@ -116,50 +126,43 @@ "pass": true }, "K3_bodyLength": { - "bodyLines": 150, + "bodyLines": 164, "pass": true }, "K5_progressiveDisclosure": { - "namedFileLinks": 0, + "namedFileLinks": 35, "folderRefs": 10, "totalRefFiles": 153, - "namedRatio": 0, - "pass": false, - "sampleNamed": [] + "namedRatio": 0.2288, + "pass": true, + "sampleNamed": [ + "references/rag-architecture/rag-core-patterns.md", + "references/rag-architecture/agentic-rag-patterns.md", + "references/rag-architecture/azure-ai-search-setup.md", + "references/rag-architecture/chunking-strategies.md", + "references/rag-architecture/hybrid-search-configuration.md" + ] }, "K6_routingTable": { - "namedStartFiles": 0, - "pass": false + "namedStartFiles": 35, + "pass": true }, "refCountConsistency": { - "consistent": false, - "mismatches": [ - { - "folder": "agent-orchestration", - "cited": 20, - "actual": 24 - }, - { - "folder": "agent-orchestration", - "citedMultiple": [ - 24, - 20 - ], - "actual": 24 - } - ] + "consistent": true, + "mismatches": [] }, "K9_timeSensitiveHints": { "timeSensitiveTokenHits": 3, "sample": [ - "GA", - "preview" + "ga", + "preview", + "GA" ] } }, "judgeInputs": { "description": "Deep technical guidance for building AI solutions in the Microsoft stack — RAG architecture, multi-agent orchestration, Azure AI Services, data engineering with Fabric, MLOps/GenAIOps, multimodal AI, API Management for AI. Triggers on: \"RAG architecture on Azure\", \"multi-agent orchestration pattern\", \"MLOps for generative AI\", \"Azure AI Search\", \"Semantic Kernel agent\", \"Fabric data pipeline\".", - "bodyLines": 150, + "bodyLines": 164, "refFileSample": [ "skills/ms-ai-engineering/references/agent-orchestration/agent-365-governance-and-deployment.md", "skills/ms-ai-engineering/references/agent-orchestration/agent-autonomy-and-control-governance.md", @@ -172,34 +175,34 @@ ] }, "judge": { + "_updated": "S11 (2026-06-20) — K1 authoritative (blinded judge vs curated 20-set); K9 prior S10", "K1_triggerPrecision": { - "provisional": true, - "precision": 0.95, - "notes": "19/20. Miss: 'Semantic Kernel agent governance' overlaps security/governance because description lists 'governance' under agent-orchestration. Operator must curate." + "provisional": false, + "inDomainHitRate": 1, + "outDomainFalsePositiveRate": 0, + "precision": 1, + "pass": true, + "misclassified": [], + "notes": "S11 authoritative: blinded judge vs operator-curated 20-set. 10/10 in-domain hits, 0/10 false positives. Clean sibling separation (advisor/governance/security/infra/off-topic not triggered)." }, "K4_noDuplication": { - "score": 4, + "score": 5, "pass": true, - "evidence": "Mostly router. Two inline tables (RAG-vs-finetuning; MLOps test-types w/ Ragas/red-teaming) restate comparison content also in ref files — summary-level, not verbatim." + "evidence": "S10 re-judge: 7 section intros are orientation prose routing to references// + named kjernefiler; the two body tables (RAG-vs-finetuning, MLOps test-types) have no verbatim row-match in refs. No duplication." }, "K7_imperativeStyle": { - "ratio": 0.9, + "ratio": 1, "pass": true, - "notes": "9/10; section intros are descriptive prose by design." + "notes": "10/10 sampled instruction sentences imperative." }, "K8_sourceCitation": { "ratio": 1, "pass": true, - "notes": "5/5 dated headers; schema drift ('Dato:' vs 'Last updated'); no source-URL in headers." + "notes": "5/5 dated headers across 5 domains; format inconsistent (EN/NO, month vs day granularity)." }, "K9_noTimeSensitive": { - "pass": false, - "findings": [ - "Line 119: 'Foundry Agent Service GA' — explicit GA-status claim in body", - "GPT-4o/Whisper/text-embedding-3/Florence — version-pinned product names in body", - "Line 48: 'Agent Framework (erstatter Semantic Kernel Agents)' — lifecycle/transition claim", - "A2A/CUA/Foundry Workflows — era-bound feature names" - ] + "pass": true, + "findings": [] } } }, @@ -214,20 +217,20 @@ }, "deterministic": { "K2_descriptionFormat": { - "quotedPhrases": 6, + "quotedPhrases": 8, "useWhenForm": true, "imperativeStart": false, "pass": true }, "K3_bodyLength": { - "bodyLines": 299, + "bodyLines": 289, "pass": true }, "K5_progressiveDisclosure": { - "namedFileLinks": 24, + "namedFileLinks": 25, "folderRefs": 5, "totalRefFiles": 78, - "namedRatio": 0.3077, + "namedRatio": 0.3205, "pass": true, "sampleNamed": [ "references/norwegian-public-sector-governance/utredningsinstruksen-ai-methodology.md", @@ -238,18 +241,12 @@ ] }, "K6_routingTable": { - "namedStartFiles": 24, + "namedStartFiles": 25, "pass": true }, "refCountConsistency": { - "consistent": false, - "mismatches": [ - { - "folder": "norwegian-public-sector-governance", - "cited": 29, - "actual": 30 - } - ] + "consistent": true, + "mismatches": [] }, "K9_timeSensitiveHints": { "timeSensitiveTokenHits": 1, @@ -259,8 +256,8 @@ } }, "judgeInputs": { - "description": "Norwegian public sector AI compliance, utredningsinstruksen for AI, EU AI Act risk classification, DPIA for AI systems, Digdir architecture principles, responsible AI governance, monitoring and observability. Triggers on: \"Norwegian public sector AI compliance\", \"AI Act risk classification\", \"DPIA for AI system\", \"Digdir architecture principles\", \"ansvarlig AI i offentlig sektor\", \"Forvaltningsloven AI\".", - "bodyLines": 299, + "description": "Norwegian public sector AI compliance, utredningsinstruksen for AI, EU AI Act risk classification, DPIA for AI systems, cross-border personal-data transfer assessment (Schrems II / TIA), Digdir architecture principles, responsible AI governance, monitoring and observability. Triggers on: \"Norwegian public sector AI compliance\", \"AI Act risk classification\", \"DPIA for AI system\", \"Schrems II data transfer\", \"overføring av persondata til tredjeland\", \"Digdir architecture principles\", \"ansvarlig AI i offentlig sektor\", \"Forvaltningsloven AI\".", + "bodyLines": 289, "refFileSample": [ "skills/ms-ai-governance/references/monitoring-observability/alerting-strategies-escalation.md", "skills/ms-ai-governance/references/monitoring-observability/anomaly-detection-ai-systems.md", @@ -273,33 +270,34 @@ ] }, "judge": { + "_updated": "S11 (2026-06-20) — K1 authoritative (blinded judge vs curated 20-set); Schrems II trigger added; K4+K9 prior S9", "K1_triggerPrecision": { - "provisional": true, - "precision": 0.95, - "notes": "19/20. Miss: Schrems II / data-transfer — body section 2.3 covers it but description has NO Schrems/dataoverføring keyword. Add trigger phrase to description." + "provisional": false, + "inDomainHitRate": 1, + "outDomainFalsePositiveRate": 0, + "precision": 1, + "pass": true, + "misclassified": [], + "notes": "S11 authoritative: blinded judge vs operator-curated 20-set. 10/10 in-domain hits, 0/10 false positives (was 0.85). Schrems II recall fix VALIDATED — judge triggered on Schrems II / overføring-til-tredjeland prompts via the two new trigger phrases." }, "K4_noDuplication": { - "score": 3, - "pass": false, - "evidence": "Moderate duplication: §6.1 DPIA risk-factor tree (incl. >=2-faktorer threshold) duplicates dpia-norwegian-methodology-ai.md; §6.2 + §2.1 AI Act taxonomy duplicates ai-act-classification-methodology.md; §1.2 Digdir 7-principle table restates per-principle files. Regulatory update must be applied twice." + "score": 5, + "pass": true, + "evidence": "S9 FIX: §6.2 now a compact decision-flow with explicit pointers ('[full forbudsliste i ai-act-classification-methodology.md]', '[åtte kategorier ...]') — Art.5 + Annex III lists no longer enumerated in body; they live only in references/responsible-ai/ai-act-classification-methodology.md. §2.1 is the single 4-level overview table ('ikke gjenta dem her'). §6.1 (DPIA tree), §1.2 (Digdir table), §6.3/§6.4 are routing/orientation, not verbatim copies of ref files." }, "K7_imperativeStyle": { "ratio": 1, "pass": true, - "notes": "10/10 imperative." + "notes": "10/10 sampled instruction sentences imperative/infinitive." }, "K8_sourceCitation": { "ratio": 1, "pass": true, - "notes": "5/5 dated headers + Status + inline source citations (Lovdata/NSM/europalov). BONUS DEFECT: SKILL.md line 191 references 'drift-detection-automated-retraining.md' which does NOT exist (actual: model-performance-drift-detection.md) — broken ref path." + "notes": "5/5 sampled refs carry Last updated + Status + Category headers." }, "K9_noTimeSensitive": { - "pass": false, - "findings": [ - "Line 121: 'Microsoft EU Data Boundary ... Azure OpenAI (Sweden Central, West Europe)' — volatile region/availability claim in body (most genuine finding)", - "Lines 81/121: 'Regulation 2024/1689' + 'Schrems II (C-311/18)' — legal identifiers (stable, borderline)", - "§2.1/§6.2 AI Act obligation status presented without Digital Omnibus caveat — may date" - ] + "pass": true, + "findings": [] } } }, @@ -319,7 +317,7 @@ "pass": true }, "K3_bodyLength": { - "bodyLines": 290, + "bodyLines": 279, "pass": true }, "K5_progressiveDisclosure": { @@ -345,17 +343,16 @@ "mismatches": [] }, "K9_timeSensitiveHints": { - "timeSensitiveTokenHits": 7, + "timeSensitiveTokenHits": 6, "sample": [ "preview", - "GA", - "v2" + "GA" ] } }, "judgeInputs": { "description": "Disaster recovery for AI workloads, multi-region Azure AI deployment, hybrid or edge AI architecture, sovereign cloud for Norway, offline-first AI patterns, AI infrastructure resilience. Covers BCDR, Azure Arc for AI, ONNX Runtime edge deployment, disconnected scenarios, Norwegian data sovereignty. Triggers on: \"disaster recovery for AI workloads\", \"edge AI deployment\", \"sovereign cloud AI\", \"Azure Arc for AI\", \"BCDR for AI\".", - "bodyLines": 290, + "bodyLines": 279, "refFileSample": [ "skills/ms-ai-infrastructure/references/bcdr/ai-foundry-disaster-recovery-planning.md", "skills/ms-ai-infrastructure/references/bcdr/backup-recovery-strategies-ai-workloads.md", @@ -368,34 +365,34 @@ ] }, "judge": { + "_updated": "S11 (2026-06-20) — K1 authoritative (blinded judge vs curated 20-set); K9 prior S10", "K1_triggerPrecision": { - "provisional": true, + "provisional": false, + "inDomainHitRate": 1, + "outDomainFalsePositiveRate": 0, "precision": 1, - "notes": "20/20. Tightly scoped + named trigger phrases, low false-positive risk. Borderline (hybrid RAG, multi-region cost) need real-corpus validation. Operator must curate." + "pass": true, + "misclassified": [], + "notes": "S11 authoritative: blinded judge vs operator-curated 20-set. 10/10 in-domain hits, 0/10 false positives. BCDR/edge/sovereign/hybrid in-domain all triggered; engineering/governance/security/advisor/cost correctly excluded." }, "K4_noDuplication": { - "score": 4, + "score": 5, "pass": true, - "evidence": "Largely summary/routing w/ '> Ref:' deferral. Minor: RTO/RPO table + SLA table put specific values in body that also live in ref files." + "evidence": "S10 re-judge: consistent summary+pointer pattern; §1.2 RTO/RPO now ~2 lines delegating to bcdr/rto-rpo-planning-ai-services.md (265 lines). SLA table replaced by relative-guidance prose. No procedural duplication." }, "K7_imperativeStyle": { "ratio": 1, "pass": true, - "notes": "10/10 imperative (concentrated in per-section directives)." + "notes": "10/10 sampled instruction sentences imperative/infinitive." }, "K8_sourceCitation": { "ratio": 1, "pass": true, - "notes": "5/5 'Last updated: 2026-02' + Status. NO source-URL in headers; identical batch date reads as generation timestamp, not per-file verification." + "notes": "5/5 'Last updated: 2026-02' + Status; no source-URL on header line." }, "K9_noTimeSensitive": { - "pass": false, - "findings": [ - "Lines 93-98: SLA table w/ hardcoded percentages (99.9/99.999/99.95%) + 'Standard v2'", - "Lines 149/158/184/186/249: Phi-3/Phi-4 model versions + param counts (3.8B/14B) in body", - "Line 141: 'Azure Local (tidl. Azure Stack HCI)' rename note", - "Line 87: hardcoded 'peak + 30% buffer'" - ] + "pass": true, + "findings": [] } } }, @@ -416,25 +413,25 @@ "pass": true }, "K3_bodyLength": { - "bodyLines": 212, + "bodyLines": 191, "pass": true }, "K5_progressiveDisclosure": { - "namedFileLinks": 10, + "namedFileLinks": 20, "folderRefs": 6, "totalRefFiles": 62, - "namedRatio": 0.1613, - "pass": false, + "namedRatio": 0.3226, + "pass": true, "sampleNamed": [ "references/ai-security-engineering/security-scoring-rubrics-6x5.md", "references/ai-security-engineering/ai-security-scoring-framework.md", - "references/ai-security-engineering/secure-model-deployment-hardening.md", - "references/ai-security-engineering/zero-trust-ai-services.md", - "references/cost-optimization/deterministic-cost-calculation-model.md" + "references/ai-security-engineering/owasp-llm-top10-azure-mitigations.md", + "references/ai-security-engineering/defender-threat-protection-ai-services.md", + "references/ai-security-engineering/secure-model-deployment-hardening.md" ] }, "K6_routingTable": { - "namedStartFiles": 10, + "namedStartFiles": 20, "pass": true }, "refCountConsistency": { @@ -442,17 +439,15 @@ "mismatches": [] }, "K9_timeSensitiveHints": { - "timeSensitiveTokenHits": 3, + "timeSensitiveTokenHits": 1, "sample": [ - "2025", - "GA", - "Preview" + "2025" ] } }, "judgeInputs": { "description": "Security assessment, cost estimation, OWASP LLM Top 10 mitigations, performance optimization for AI on Microsoft stack. Deterministic 6x5 security scoring, P10/P50/P90 cost confidence intervals, FinOps practices. Triggers on: \"security assessment for AI\", \"AI threat modeling\", \"cost estimation for Azure AI\", \"FinOps for AI workloads\", \"OWASP LLM\", \"kostnadsestimat for AI-løsning\".", - "bodyLines": 212, + "bodyLines": 191, "refFileSample": [ "skills/ms-ai-security/references/ai-security-engineering/adversarial-input-robustness-testing.md", "skills/ms-ai-security/references/ai-security-engineering/ai-incident-response-procedures.md", @@ -465,34 +460,34 @@ ] }, "judge": { + "_updated": "S11 (2026-06-20) — K1 authoritative (blinded judge vs curated 20-set); K4+K9 prior S9", "K1_triggerPrecision": { - "provisional": true, - "precision": 0.95, - "notes": "19/20. Clean sibling separation (AI Act/DPIA/platform/RAG/BCDR not triggered). One ambiguity: model feature-comparison could be pulled by broad 'performance optimization for AI'. Operator must curate." + "provisional": false, + "inDomainHitRate": 1, + "outDomainFalsePositiveRate": 0, + "precision": 1, + "pass": true, + "misclassified": [], + "notes": "S11 authoritative: blinded judge vs operator-curated 20-set. 10/10 in-domain hits, 0/10 false positives. Borderline content-safety/TCO correctly included; sibling governance/engineering/infra/advisor correctly excluded." }, "K4_noDuplication": { - "score": 3, - "pass": false, - "evidence": "DUPLICATION-WITH-CONTRADICTION: body weighting table (L52-59 Standard: Identity 20/Network 15/Data 20/Content 20/Compliance 15/Monitoring 10) CONTRADICTS canonical rubric security-scoring-rubrics-6x5.md (Compliance 25/Data 20/Identity 20/Content 15/Network 10/Monitoring 10). Body scoring rule (weighted sum) also diverges from rubric (Ja-checkpoint count). Real correctness bug." + "score": 5, + "pass": true, + "evidence": "S9 re-judge (cold, post-fix): (a) 6x5 weights — body L50 routes to security-scoring-rubrics-6x5.md ('Ikke dupliser vekttallene her'), no body numbers; (b) risk-classification thresholds — body L54 routes to same rubric ('Ikke dupliser terskeltallene her'), canonical mapping incl. 1.00-1.49 Uakseptabel lives only in rubric; (c) P10/P50/P90 — body L94 affirms 'per komponent (ikke flat multiplikator)' owned by deterministic-cost-calculation-model.md §3, concrete factors only in cost model — body affirms, does not contradict; (d) OWASP table + §3 perf are routing/orientation with explicit volatile-numbers-live-in-refs note. No duplication/contradiction." }, "K7_imperativeStyle": { "ratio": 1, "pass": true, - "notes": "10/10 imperative." + "notes": "10/10 sampled instruction sentences imperative/infinitive." }, "K8_sourceCitation": { "ratio": 1, "pass": true, - "notes": "5/5 dated headers; mostly no header source-URL (URLs in body cost-register rows)." + "notes": "5/5 dated headers + Status; 3/5 also carry Verified: MCP ." }, "K9_noTimeSensitive": { - "pass": false, - "findings": [ - "Line 73: 'OWASP LLM Top 10 (2025)' — dated standard version", - "Line 92: 'Defender ... GA for AI applications, Preview for AI agents' + 'ikke i Azure Government' — GA/preview + region status in body", - "Line 138: 'GPT-4o mini vs GPT-4o' — model versions", - "Lines 149-153: hardcoded perf/price figures (20-50ms, 5-10x, 50%/80%, Batch 50% @ 24h SLA)" - ] + "pass": true, + "findings": [] } } }