Kjell Tore Guttormsen
ba597eb988
- Refaktor: overlap-kjerne flyttet til scripts/kb-eval/lib/sibling-overlap.mjs (bryter sirkulær import eval.mjs<->detect); detect re-eksporterer → B1-tester urørt - K10 = søsken-scope-ikke-overlapp, deterministisk cross-skill: perSkillSiblingOverlap + attachSiblingOverlap i eval.mjs. combined = boundaryTension + df-vektet leksikalsk; per-skill verdikt = verste søskenpar; terskel 7.0 (naturlig gap 7.42→6.67) - Empirisk (alle 5): eng+infra FAIL (7.42 mot hverandre), advisor/gov/sec PASS → eng↔infra-signal mater B3 merge/saner (ikke blokkering) - Gated baseline-regen via --write (descriptions urørt → judge K1/K4/K7/K8/K9 merget uendret, ikke fabrikkert); rubric K1-K10 - TDD: +9 tester (tests/kb-eval/test-k10-sibling-overlap.test.mjs), kb-eval 31→40 - 0 skriving til skills/. Suiter: validate 239 · kb-update 122 · kb-integrity 192/192
525 lines
23 KiB
JSON
525 lines
23 KiB
JSON
{
|
|
"rubric": "K1-K10",
|
|
"note": "Deterministic: K2,K3,K5,K6,refCountConsistency,K10(siblingScopeOverlap). LLM-judge (operator-gated): K1,K4,K7,K8,K9.",
|
|
"skills": [
|
|
{
|
|
"name": "ms-ai-advisor",
|
|
"skillMd": "skills/ms-ai-advisor/SKILL.md",
|
|
"refFilesActual": 62,
|
|
"refCountsPerFolder": {
|
|
"architecture": 16,
|
|
"copilot-extensibility": 22,
|
|
"development": 1,
|
|
"platforms": 5,
|
|
"prompt-engineering": 18
|
|
},
|
|
"deterministic": {
|
|
"K2_descriptionFormat": {
|
|
"quotedPhrases": 6,
|
|
"useWhenForm": true,
|
|
"imperativeStart": false,
|
|
"pass": true
|
|
},
|
|
"K3_bodyLength": {
|
|
"bodyLines": 245,
|
|
"pass": true
|
|
},
|
|
"K5_progressiveDisclosure": {
|
|
"namedFileLinks": 25,
|
|
"folderRefs": 5,
|
|
"totalRefFiles": 62,
|
|
"namedRatio": 0.4032,
|
|
"pass": true,
|
|
"sampleNamed": [
|
|
"references/architecture/diagram-prompt-templates.md",
|
|
"references/platforms/azure-ai-foundry.md",
|
|
"references/platforms/m365-copilot.md",
|
|
"references/platforms/copilot-studio.md",
|
|
"references/platforms/power-platform.md"
|
|
]
|
|
},
|
|
"K6_routingTable": {
|
|
"namedStartFiles": 25,
|
|
"pass": true
|
|
},
|
|
"refCountConsistency": {
|
|
"consistent": true,
|
|
"mismatches": []
|
|
},
|
|
"K9_timeSensitiveHints": {
|
|
"timeSensitiveTokenHits": 8,
|
|
"sample": [
|
|
"Preview",
|
|
"GA",
|
|
"preview",
|
|
"2026",
|
|
"v3.0"
|
|
]
|
|
},
|
|
"K10_siblingScopeOverlap": {
|
|
"maxCombined": 4.5833,
|
|
"worstSibling": "ms-ai-engineering",
|
|
"pass": true,
|
|
"threshold": 7
|
|
}
|
|
},
|
|
"judgeInputs": {
|
|
"description": "Microsoft AI platform selection — choosing between Azure AI Foundry, M365 Copilot, Copilot Studio, Power Platform, Azure OpenAI, and Microsoft Agent Framework for a given scenario. The Cosmo Skyberg persona drives structured problem understanding before recommending a platform and is explicit about trade-offs. Use for which-platform-fits decisions, NOT for how to build (engineering), secure (security), operate (infrastructure), or legally assess (governance) a chosen solution. Triggers on: \"which Microsoft AI platform\", \"Copilot vs Foundry\", \"Copilot Studio or Azure AI Foundry\", \"help me choose an AI platform\", \"Cosmo\", \"/architect\".",
|
|
"bodyLines": 245,
|
|
"refFileSample": [
|
|
"skills/ms-ai-advisor/references/architecture/adr-template.md",
|
|
"skills/ms-ai-advisor/references/architecture/ai-utredning-template.md",
|
|
"skills/ms-ai-advisor/references/architecture/alternativanalyse-methodology.md",
|
|
"skills/ms-ai-advisor/references/architecture/capacity-feasibility-benchmarks.md",
|
|
"skills/ms-ai-advisor/references/architecture/cost-models.md",
|
|
"skills/ms-ai-advisor/references/architecture/decision-trees.md",
|
|
"skills/ms-ai-advisor/references/architecture/diagram-prompt-templates.md",
|
|
"skills/ms-ai-advisor/references/architecture/licensing-matrix.md"
|
|
]
|
|
},
|
|
"judge": {
|
|
"_updated": "S11 (2026-06-20) — K1 authoritative (blinded judge vs curated 20-set); description tightened against over-trigging",
|
|
"K1_triggerPrecision": {
|
|
"provisional": false,
|
|
"inDomainHitRate": 1,
|
|
"outDomainFalsePositiveRate": 0,
|
|
"precision": 1,
|
|
"pass": true,
|
|
"misclassified": [],
|
|
"notes": "S11 authoritative: blinded judge vs operator-curated 20-set. 10/10 in-domain hits, 0/10 false positives. Over-trigging fix VALIDATED — judge cited the new 'NOT for build/secure/operate/legally-assess' exclusion when rejecting all four sibling-domain prompts (engineering/security/governance/infra)."
|
|
},
|
|
"K4_noDuplication": {
|
|
"score": 5,
|
|
"pass": true,
|
|
"evidence": "Body = persona + 7-phase workflow + ref-index; no ref detail reproduced. Only internal MCP-table redundancy (SKILL-internal, not SKILL<->ref)."
|
|
},
|
|
"K7_imperativeStyle": {
|
|
"ratio": 1,
|
|
"pass": true,
|
|
"notes": "10/10 sampled instruction sentences imperative."
|
|
},
|
|
"K8_sourceCitation": {
|
|
"ratio": 0.8,
|
|
"pass": true,
|
|
"notes": "AT THRESHOLD: architecture/decision-trees.md lacks dated header (footer source only). Add dated header to harden margin."
|
|
},
|
|
"K9_noTimeSensitive": {
|
|
"pass": true,
|
|
"findings": [
|
|
"Only meta-instructions (preview/GA as dynamic-to-verify) + stable identifiers (M365 SKUs, MADR v3.0). No stale-able product claim in body."
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "ms-ai-engineering",
|
|
"skillMd": "skills/ms-ai-engineering/SKILL.md",
|
|
"refFilesActual": 153,
|
|
"refCountsPerFolder": {
|
|
"agent-orchestration": 24,
|
|
"api-management": 19,
|
|
"azure-ai-services": 20,
|
|
"data-engineering": 22,
|
|
"mlops-genaiops": 22,
|
|
"multi-modal": 18,
|
|
"rag-architecture": 28
|
|
},
|
|
"deterministic": {
|
|
"K2_descriptionFormat": {
|
|
"quotedPhrases": 6,
|
|
"useWhenForm": true,
|
|
"imperativeStart": false,
|
|
"pass": true
|
|
},
|
|
"K3_bodyLength": {
|
|
"bodyLines": 164,
|
|
"pass": true
|
|
},
|
|
"K5_progressiveDisclosure": {
|
|
"namedFileLinks": 35,
|
|
"folderRefs": 10,
|
|
"totalRefFiles": 153,
|
|
"namedRatio": 0.2288,
|
|
"pass": true,
|
|
"sampleNamed": [
|
|
"references/rag-architecture/rag-core-patterns.md",
|
|
"references/rag-architecture/agentic-rag-patterns.md",
|
|
"references/rag-architecture/azure-ai-search-setup.md",
|
|
"references/rag-architecture/chunking-strategies.md",
|
|
"references/rag-architecture/hybrid-search-configuration.md"
|
|
]
|
|
},
|
|
"K6_routingTable": {
|
|
"namedStartFiles": 35,
|
|
"pass": true
|
|
},
|
|
"refCountConsistency": {
|
|
"consistent": true,
|
|
"mismatches": []
|
|
},
|
|
"K9_timeSensitiveHints": {
|
|
"timeSensitiveTokenHits": 3,
|
|
"sample": [
|
|
"ga",
|
|
"preview",
|
|
"GA"
|
|
]
|
|
},
|
|
"K10_siblingScopeOverlap": {
|
|
"maxCombined": 7.4167,
|
|
"worstSibling": "ms-ai-infrastructure",
|
|
"pass": false,
|
|
"threshold": 7
|
|
}
|
|
},
|
|
"judgeInputs": {
|
|
"description": "Deep technical guidance for building AI solutions in the Microsoft stack — RAG architecture, multi-agent orchestration, Azure AI Services, data engineering with Fabric, MLOps/GenAIOps, multimodal AI, API Management for AI. Triggers on: \"RAG architecture on Azure\", \"multi-agent orchestration pattern\", \"MLOps for generative AI\", \"Azure AI Search\", \"Semantic Kernel agent\", \"Fabric data pipeline\".",
|
|
"bodyLines": 164,
|
|
"refFileSample": [
|
|
"skills/ms-ai-engineering/references/agent-orchestration/agent-365-governance-and-deployment.md",
|
|
"skills/ms-ai-engineering/references/agent-orchestration/agent-autonomy-and-control-governance.md",
|
|
"skills/ms-ai-engineering/references/agent-orchestration/agent-compliance-and-audit-trails.md",
|
|
"skills/ms-ai-engineering/references/agent-orchestration/agent-cost-optimization-strategies.md",
|
|
"skills/ms-ai-engineering/references/agent-orchestration/agent-ecosystem-and-plugin-marketplace.md",
|
|
"skills/ms-ai-engineering/references/agent-orchestration/agent-evaluation-testing-frameworks.md",
|
|
"skills/ms-ai-engineering/references/agent-orchestration/agent-feedback-and-learning-loops.md",
|
|
"skills/ms-ai-engineering/references/agent-orchestration/agent-latency-optimization.md"
|
|
]
|
|
},
|
|
"judge": {
|
|
"_updated": "S11 (2026-06-20) — K1 authoritative (blinded judge vs curated 20-set); K9 prior S10",
|
|
"K1_triggerPrecision": {
|
|
"provisional": false,
|
|
"inDomainHitRate": 1,
|
|
"outDomainFalsePositiveRate": 0,
|
|
"precision": 1,
|
|
"pass": true,
|
|
"misclassified": [],
|
|
"notes": "S11 authoritative: blinded judge vs operator-curated 20-set. 10/10 in-domain hits, 0/10 false positives. Clean sibling separation (advisor/governance/security/infra/off-topic not triggered)."
|
|
},
|
|
"K4_noDuplication": {
|
|
"score": 5,
|
|
"pass": true,
|
|
"evidence": "S10 re-judge: 7 section intros are orientation prose routing to references/<domain>/ + named kjernefiler; the two body tables (RAG-vs-finetuning, MLOps test-types) have no verbatim row-match in refs. No duplication."
|
|
},
|
|
"K7_imperativeStyle": {
|
|
"ratio": 1,
|
|
"pass": true,
|
|
"notes": "10/10 sampled instruction sentences imperative."
|
|
},
|
|
"K8_sourceCitation": {
|
|
"ratio": 1,
|
|
"pass": true,
|
|
"notes": "5/5 dated headers across 5 domains; format inconsistent (EN/NO, month vs day granularity)."
|
|
},
|
|
"K9_noTimeSensitive": {
|
|
"pass": true,
|
|
"findings": []
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "ms-ai-governance",
|
|
"skillMd": "skills/ms-ai-governance/SKILL.md",
|
|
"refFilesActual": 78,
|
|
"refCountsPerFolder": {
|
|
"monitoring-observability": 18,
|
|
"norwegian-public-sector-governance": 30,
|
|
"responsible-ai": 30
|
|
},
|
|
"deterministic": {
|
|
"K2_descriptionFormat": {
|
|
"quotedPhrases": 8,
|
|
"useWhenForm": true,
|
|
"imperativeStart": false,
|
|
"pass": true
|
|
},
|
|
"K3_bodyLength": {
|
|
"bodyLines": 289,
|
|
"pass": true
|
|
},
|
|
"K5_progressiveDisclosure": {
|
|
"namedFileLinks": 25,
|
|
"folderRefs": 5,
|
|
"totalRefFiles": 78,
|
|
"namedRatio": 0.3205,
|
|
"pass": true,
|
|
"sampleNamed": [
|
|
"references/norwegian-public-sector-governance/utredningsinstruksen-ai-methodology.md",
|
|
"references/norwegian-public-sector-governance/forvaltningsloven-ai-decisions.md",
|
|
"references/norwegian-public-sector-governance/samfunnsokonomisk-analyse-nnv.md",
|
|
"references/norwegian-public-sector-governance/gevinstrealisering-dfo-methodology.md",
|
|
"references/norwegian-public-sector-governance/anskaffelser-ai-procurement-framework.md"
|
|
]
|
|
},
|
|
"K6_routingTable": {
|
|
"namedStartFiles": 25,
|
|
"pass": true
|
|
},
|
|
"refCountConsistency": {
|
|
"consistent": true,
|
|
"mismatches": []
|
|
},
|
|
"K9_timeSensitiveHints": {
|
|
"timeSensitiveTokenHits": 1,
|
|
"sample": [
|
|
"2024"
|
|
]
|
|
},
|
|
"K10_siblingScopeOverlap": {
|
|
"maxCombined": 6.6667,
|
|
"worstSibling": "ms-ai-engineering",
|
|
"pass": true,
|
|
"threshold": 7
|
|
}
|
|
},
|
|
"judgeInputs": {
|
|
"description": "Norwegian public sector AI compliance, utredningsinstruksen for AI, EU AI Act risk classification, DPIA for AI systems, cross-border personal-data transfer assessment (Schrems II / TIA), Digdir architecture principles, responsible AI governance, monitoring and observability. Triggers on: \"Norwegian public sector AI compliance\", \"AI Act risk classification\", \"DPIA for AI system\", \"Schrems II data transfer\", \"overføring av persondata til tredjeland\", \"Digdir architecture principles\", \"ansvarlig AI i offentlig sektor\", \"Forvaltningsloven AI\".",
|
|
"bodyLines": 289,
|
|
"refFileSample": [
|
|
"skills/ms-ai-governance/references/monitoring-observability/alerting-strategies-escalation.md",
|
|
"skills/ms-ai-governance/references/monitoring-observability/anomaly-detection-ai-systems.md",
|
|
"skills/ms-ai-governance/references/monitoring-observability/application-insights-llm-monitoring.md",
|
|
"skills/ms-ai-governance/references/monitoring-observability/azure-monitor-setup-ai-workloads.md",
|
|
"skills/ms-ai-governance/references/monitoring-observability/compliance-monitoring-ai-governance.md",
|
|
"skills/ms-ai-governance/references/monitoring-observability/cost-monitoring-cost-attribution.md",
|
|
"skills/ms-ai-governance/references/monitoring-observability/custom-dashboards-ai-operations.md",
|
|
"skills/ms-ai-governance/references/monitoring-observability/data-residency-audit-monitoring.md"
|
|
]
|
|
},
|
|
"judge": {
|
|
"_updated": "S11 (2026-06-20) — K1 authoritative (blinded judge vs curated 20-set); Schrems II trigger added; K4+K9 prior S9",
|
|
"K1_triggerPrecision": {
|
|
"provisional": false,
|
|
"inDomainHitRate": 1,
|
|
"outDomainFalsePositiveRate": 0,
|
|
"precision": 1,
|
|
"pass": true,
|
|
"misclassified": [],
|
|
"notes": "S11 authoritative: blinded judge vs operator-curated 20-set. 10/10 in-domain hits, 0/10 false positives (was 0.85). Schrems II recall fix VALIDATED — judge triggered on Schrems II / overføring-til-tredjeland prompts via the two new trigger phrases."
|
|
},
|
|
"K4_noDuplication": {
|
|
"score": 5,
|
|
"pass": true,
|
|
"evidence": "S9 FIX: §6.2 now a compact decision-flow with explicit pointers ('[full forbudsliste i ai-act-classification-methodology.md]', '[åtte kategorier ...]') — Art.5 + Annex III lists no longer enumerated in body; they live only in references/responsible-ai/ai-act-classification-methodology.md. §2.1 is the single 4-level overview table ('ikke gjenta dem her'). §6.1 (DPIA tree), §1.2 (Digdir table), §6.3/§6.4 are routing/orientation, not verbatim copies of ref files."
|
|
},
|
|
"K7_imperativeStyle": {
|
|
"ratio": 1,
|
|
"pass": true,
|
|
"notes": "10/10 sampled instruction sentences imperative/infinitive."
|
|
},
|
|
"K8_sourceCitation": {
|
|
"ratio": 1,
|
|
"pass": true,
|
|
"notes": "5/5 sampled refs carry Last updated + Status + Category headers."
|
|
},
|
|
"K9_noTimeSensitive": {
|
|
"pass": true,
|
|
"findings": []
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "ms-ai-infrastructure",
|
|
"skillMd": "skills/ms-ai-infrastructure/SKILL.md",
|
|
"refFilesActual": 34,
|
|
"refCountsPerFolder": {
|
|
"bcdr": 16,
|
|
"hybrid-edge": 18
|
|
},
|
|
"deterministic": {
|
|
"K2_descriptionFormat": {
|
|
"quotedPhrases": 5,
|
|
"useWhenForm": true,
|
|
"imperativeStart": false,
|
|
"pass": true
|
|
},
|
|
"K3_bodyLength": {
|
|
"bodyLines": 279,
|
|
"pass": true
|
|
},
|
|
"K5_progressiveDisclosure": {
|
|
"namedFileLinks": 33,
|
|
"folderRefs": 5,
|
|
"totalRefFiles": 34,
|
|
"namedRatio": 0.9706,
|
|
"pass": true,
|
|
"sampleNamed": [
|
|
"references/bcdr/multi-region-azure-openai-deployment.md",
|
|
"references/bcdr/rto-rpo-planning-ai-services.md",
|
|
"references/bcdr/backup-recovery-strategies-ai-workloads.md",
|
|
"references/bcdr/failover-testing-ai-services.md",
|
|
"references/bcdr/chaos-engineering-ai-systems.md"
|
|
]
|
|
},
|
|
"K6_routingTable": {
|
|
"namedStartFiles": 33,
|
|
"pass": true
|
|
},
|
|
"refCountConsistency": {
|
|
"consistent": true,
|
|
"mismatches": []
|
|
},
|
|
"K9_timeSensitiveHints": {
|
|
"timeSensitiveTokenHits": 6,
|
|
"sample": [
|
|
"preview",
|
|
"GA"
|
|
]
|
|
},
|
|
"K10_siblingScopeOverlap": {
|
|
"maxCombined": 7.4167,
|
|
"worstSibling": "ms-ai-engineering",
|
|
"pass": false,
|
|
"threshold": 7
|
|
}
|
|
},
|
|
"judgeInputs": {
|
|
"description": "Disaster recovery for AI workloads, multi-region Azure AI deployment, hybrid or edge AI architecture, sovereign cloud for Norway, offline-first AI patterns, AI infrastructure resilience. Covers BCDR, Azure Arc for AI, ONNX Runtime edge deployment, disconnected scenarios, Norwegian data sovereignty. Triggers on: \"disaster recovery for AI workloads\", \"edge AI deployment\", \"sovereign cloud AI\", \"Azure Arc for AI\", \"BCDR for AI\".",
|
|
"bodyLines": 279,
|
|
"refFileSample": [
|
|
"skills/ms-ai-infrastructure/references/bcdr/ai-foundry-disaster-recovery-planning.md",
|
|
"skills/ms-ai-infrastructure/references/bcdr/backup-recovery-strategies-ai-workloads.md",
|
|
"skills/ms-ai-infrastructure/references/bcdr/capacity-planning-dr-configurations.md",
|
|
"skills/ms-ai-infrastructure/references/bcdr/chaos-engineering-ai-systems.md",
|
|
"skills/ms-ai-infrastructure/references/bcdr/compliance-requirements-bcdr.md",
|
|
"skills/ms-ai-infrastructure/references/bcdr/cost-analysis-dr-configurations.md",
|
|
"skills/ms-ai-infrastructure/references/bcdr/data-replication-patterns-ai.md",
|
|
"skills/ms-ai-infrastructure/references/bcdr/failover-testing-ai-services.md"
|
|
]
|
|
},
|
|
"judge": {
|
|
"_updated": "S11 (2026-06-20) — K1 authoritative (blinded judge vs curated 20-set); K9 prior S10",
|
|
"K1_triggerPrecision": {
|
|
"provisional": false,
|
|
"inDomainHitRate": 1,
|
|
"outDomainFalsePositiveRate": 0,
|
|
"precision": 1,
|
|
"pass": true,
|
|
"misclassified": [],
|
|
"notes": "S11 authoritative: blinded judge vs operator-curated 20-set. 10/10 in-domain hits, 0/10 false positives. BCDR/edge/sovereign/hybrid in-domain all triggered; engineering/governance/security/advisor/cost correctly excluded."
|
|
},
|
|
"K4_noDuplication": {
|
|
"score": 5,
|
|
"pass": true,
|
|
"evidence": "S10 re-judge: consistent summary+pointer pattern; §1.2 RTO/RPO now ~2 lines delegating to bcdr/rto-rpo-planning-ai-services.md (265 lines). SLA table replaced by relative-guidance prose. No procedural duplication."
|
|
},
|
|
"K7_imperativeStyle": {
|
|
"ratio": 1,
|
|
"pass": true,
|
|
"notes": "10/10 sampled instruction sentences imperative/infinitive."
|
|
},
|
|
"K8_sourceCitation": {
|
|
"ratio": 1,
|
|
"pass": true,
|
|
"notes": "5/5 'Last updated: 2026-02' + Status; no source-URL on header line."
|
|
},
|
|
"K9_noTimeSensitive": {
|
|
"pass": true,
|
|
"findings": []
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "ms-ai-security",
|
|
"skillMd": "skills/ms-ai-security/SKILL.md",
|
|
"refFilesActual": 62,
|
|
"refCountsPerFolder": {
|
|
"ai-security-engineering": 22,
|
|
"cost-optimization": 22,
|
|
"performance-scalability": 18
|
|
},
|
|
"deterministic": {
|
|
"K2_descriptionFormat": {
|
|
"quotedPhrases": 6,
|
|
"useWhenForm": true,
|
|
"imperativeStart": false,
|
|
"pass": true
|
|
},
|
|
"K3_bodyLength": {
|
|
"bodyLines": 191,
|
|
"pass": true
|
|
},
|
|
"K5_progressiveDisclosure": {
|
|
"namedFileLinks": 20,
|
|
"folderRefs": 6,
|
|
"totalRefFiles": 62,
|
|
"namedRatio": 0.3226,
|
|
"pass": true,
|
|
"sampleNamed": [
|
|
"references/ai-security-engineering/security-scoring-rubrics-6x5.md",
|
|
"references/ai-security-engineering/ai-security-scoring-framework.md",
|
|
"references/ai-security-engineering/owasp-llm-top10-azure-mitigations.md",
|
|
"references/ai-security-engineering/defender-threat-protection-ai-services.md",
|
|
"references/ai-security-engineering/secure-model-deployment-hardening.md"
|
|
]
|
|
},
|
|
"K6_routingTable": {
|
|
"namedStartFiles": 20,
|
|
"pass": true
|
|
},
|
|
"refCountConsistency": {
|
|
"consistent": true,
|
|
"mismatches": []
|
|
},
|
|
"K9_timeSensitiveHints": {
|
|
"timeSensitiveTokenHits": 1,
|
|
"sample": [
|
|
"2025"
|
|
]
|
|
},
|
|
"K10_siblingScopeOverlap": {
|
|
"maxCombined": 6.5,
|
|
"worstSibling": "ms-ai-governance",
|
|
"pass": true,
|
|
"threshold": 7
|
|
}
|
|
},
|
|
"judgeInputs": {
|
|
"description": "Security assessment, cost estimation, OWASP LLM Top 10 mitigations, performance optimization for AI on Microsoft stack. Deterministic 6x5 security scoring, P10/P50/P90 cost confidence intervals, FinOps practices. Triggers on: \"security assessment for AI\", \"AI threat modeling\", \"cost estimation for Azure AI\", \"FinOps for AI workloads\", \"OWASP LLM\", \"kostnadsestimat for AI-løsning\".",
|
|
"bodyLines": 191,
|
|
"refFileSample": [
|
|
"skills/ms-ai-security/references/ai-security-engineering/adversarial-input-robustness-testing.md",
|
|
"skills/ms-ai-security/references/ai-security-engineering/ai-incident-response-procedures.md",
|
|
"skills/ms-ai-security/references/ai-security-engineering/ai-prompt-shield-network.md",
|
|
"skills/ms-ai-security/references/ai-security-engineering/ai-red-team-operations-practical.md",
|
|
"skills/ms-ai-security/references/ai-security-engineering/ai-security-scoring-framework.md",
|
|
"skills/ms-ai-security/references/ai-security-engineering/ai-threat-modeling-stride.md",
|
|
"skills/ms-ai-security/references/ai-security-engineering/content-safety-filter-calibration.md",
|
|
"skills/ms-ai-security/references/ai-security-engineering/data-leakage-prevention-ai.md"
|
|
]
|
|
},
|
|
"judge": {
|
|
"_updated": "S11 (2026-06-20) — K1 authoritative (blinded judge vs curated 20-set); K4+K9 prior S9",
|
|
"K1_triggerPrecision": {
|
|
"provisional": false,
|
|
"inDomainHitRate": 1,
|
|
"outDomainFalsePositiveRate": 0,
|
|
"precision": 1,
|
|
"pass": true,
|
|
"misclassified": [],
|
|
"notes": "S11 authoritative: blinded judge vs operator-curated 20-set. 10/10 in-domain hits, 0/10 false positives. Borderline content-safety/TCO correctly included; sibling governance/engineering/infra/advisor correctly excluded."
|
|
},
|
|
"K4_noDuplication": {
|
|
"score": 5,
|
|
"pass": true,
|
|
"evidence": "S9 re-judge (cold, post-fix): (a) 6x5 weights — body L50 routes to security-scoring-rubrics-6x5.md ('Ikke dupliser vekttallene her'), no body numbers; (b) risk-classification thresholds — body L54 routes to same rubric ('Ikke dupliser terskeltallene her'), canonical mapping incl. 1.00-1.49 Uakseptabel lives only in rubric; (c) P10/P50/P90 — body L94 affirms 'per komponent (ikke flat multiplikator)' owned by deterministic-cost-calculation-model.md §3, concrete factors only in cost model — body affirms, does not contradict; (d) OWASP table + §3 perf are routing/orientation with explicit volatile-numbers-live-in-refs note. No duplication/contradiction."
|
|
},
|
|
"K7_imperativeStyle": {
|
|
"ratio": 1,
|
|
"pass": true,
|
|
"notes": "10/10 sampled instruction sentences imperative/infinitive."
|
|
},
|
|
"K8_sourceCitation": {
|
|
"ratio": 1,
|
|
"pass": true,
|
|
"notes": "5/5 dated headers + Status; 3/5 also carry Verified: MCP <date>."
|
|
},
|
|
"K9_noTimeSensitive": {
|
|
"pass": true,
|
|
"findings": []
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|