feat(ms-ai-architect): Sesjon 1 - skill eval-baseline (rubrikk K1-K9) + ai-foundry probe
Read-only eval-verktoy scripts/kb-eval/eval.mjs (determ. K2/K3/K5/K6 + ref-tall) + operator-gated LLM-judge (judge-prompt.md -> data/judge-results.json) flettet til data/eval-baseline.json. 13 enhetstester (tests/kb-eval/), 42 kb-update-tester gronne. Baseline: K5 3/5 fail, K9 4/5 fail, K4 2/5 fail, ref-tall 2/5 fail; 2 konkrete bugs flagget (security 6x5-vekting-motstrid, governance broken ref-path). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01REiKFhP4w6xGXXqWKpPCJJ
This commit is contained in:
parent
bae0977292
commit
215772df87
5 changed files with 936 additions and 0 deletions
500
scripts/kb-eval/data/eval-baseline.json
Normal file
500
scripts/kb-eval/data/eval-baseline.json
Normal file
|
|
@ -0,0 +1,500 @@
|
|||
{
|
||||
"rubric": "K1-K9",
|
||||
"note": "Deterministic: K2,K3,K5,K6,refCountConsistency. LLM-judge (operator-gated): K1,K4,K7,K8,K9.",
|
||||
"skills": [
|
||||
{
|
||||
"name": "ms-ai-advisor",
|
||||
"skillMd": "skills/ms-ai-advisor/SKILL.md",
|
||||
"refFilesActual": 62,
|
||||
"refCountsPerFolder": {
|
||||
"architecture": 16,
|
||||
"copilot-extensibility": 22,
|
||||
"development": 1,
|
||||
"platforms": 5,
|
||||
"prompt-engineering": 18
|
||||
},
|
||||
"deterministic": {
|
||||
"K2_descriptionFormat": {
|
||||
"quotedPhrases": 5,
|
||||
"useWhenForm": true,
|
||||
"imperativeStart": false,
|
||||
"pass": true
|
||||
},
|
||||
"K3_bodyLength": {
|
||||
"bodyLines": 240,
|
||||
"pass": true
|
||||
},
|
||||
"K5_progressiveDisclosure": {
|
||||
"namedFileLinks": 1,
|
||||
"folderRefs": 5,
|
||||
"totalRefFiles": 62,
|
||||
"namedRatio": 0.0161,
|
||||
"pass": false,
|
||||
"sampleNamed": [
|
||||
"references/architecture/diagram-prompt-templates.md"
|
||||
]
|
||||
},
|
||||
"K6_routingTable": {
|
||||
"namedStartFiles": 1,
|
||||
"pass": true
|
||||
},
|
||||
"refCountConsistency": {
|
||||
"consistent": true,
|
||||
"mismatches": []
|
||||
},
|
||||
"K9_timeSensitiveHints": {
|
||||
"timeSensitiveTokenHits": 7,
|
||||
"sample": [
|
||||
"Preview",
|
||||
"GA",
|
||||
"preview",
|
||||
"v3.0"
|
||||
]
|
||||
}
|
||||
},
|
||||
"judgeInputs": {
|
||||
"description": "Microsoft AI architecture guidance, choosing between Azure AI platforms, Copilot vs Foundry trade-offs. Cosmo Skyberg persona guides through structured problem understanding before technology selection. Specialist in Azure AI Foundry, M365 Copilot, Copilot Studio, Power Platform, Azure OpenAI, Microsoft Agent Framework. Triggers on: \"Microsoft AI architecture\", \"Copilot vs Foundry\", \"which Microsoft AI platform\", \"Cosmo\", \"/architect\".",
|
||||
"bodyLines": 240,
|
||||
"refFileSample": [
|
||||
"skills/ms-ai-advisor/references/architecture/adr-template.md",
|
||||
"skills/ms-ai-advisor/references/architecture/ai-utredning-template.md",
|
||||
"skills/ms-ai-advisor/references/architecture/alternativanalyse-methodology.md",
|
||||
"skills/ms-ai-advisor/references/architecture/capacity-feasibility-benchmarks.md",
|
||||
"skills/ms-ai-advisor/references/architecture/cost-models.md",
|
||||
"skills/ms-ai-advisor/references/architecture/decision-trees.md",
|
||||
"skills/ms-ai-advisor/references/architecture/diagram-prompt-templates.md",
|
||||
"skills/ms-ai-advisor/references/architecture/licensing-matrix.md"
|
||||
]
|
||||
},
|
||||
"judge": {
|
||||
"K1_triggerPrecision": {
|
||||
"provisional": true,
|
||||
"precision": 1,
|
||||
"notes": "20/20 from description. BORDERLINE: cost/diagram/DPIA/AI-Act prompts are MS-AI-adjacent; broad 'Microsoft AI architecture' phrase risks over-triggering vs sibling commands. Operator must curate + stress-test sibling overlap."
|
||||
},
|
||||
"K4_noDuplication": {
|
||||
"score": 5,
|
||||
"pass": true,
|
||||
"evidence": "Body = persona + 7-phase workflow + ref-index; no ref detail reproduced. Only internal MCP-table redundancy (SKILL-internal, not SKILL<->ref)."
|
||||
},
|
||||
"K7_imperativeStyle": {
|
||||
"ratio": 1,
|
||||
"pass": true,
|
||||
"notes": "10/10 sampled instruction sentences imperative."
|
||||
},
|
||||
"K8_sourceCitation": {
|
||||
"ratio": 0.8,
|
||||
"pass": true,
|
||||
"notes": "AT THRESHOLD: architecture/decision-trees.md lacks dated header (footer source only). Add dated header to harden margin."
|
||||
},
|
||||
"K9_noTimeSensitive": {
|
||||
"pass": true,
|
||||
"findings": [
|
||||
"Only meta-instructions (preview/GA as dynamic-to-verify) + stable identifiers (M365 SKUs, MADR v3.0). No stale-able product claim in body."
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ms-ai-engineering",
|
||||
"skillMd": "skills/ms-ai-engineering/SKILL.md",
|
||||
"refFilesActual": 153,
|
||||
"refCountsPerFolder": {
|
||||
"agent-orchestration": 24,
|
||||
"api-management": 19,
|
||||
"azure-ai-services": 20,
|
||||
"data-engineering": 22,
|
||||
"mlops-genaiops": 22,
|
||||
"multi-modal": 18,
|
||||
"rag-architecture": 28
|
||||
},
|
||||
"deterministic": {
|
||||
"K2_descriptionFormat": {
|
||||
"quotedPhrases": 6,
|
||||
"useWhenForm": true,
|
||||
"imperativeStart": false,
|
||||
"pass": true
|
||||
},
|
||||
"K3_bodyLength": {
|
||||
"bodyLines": 150,
|
||||
"pass": true
|
||||
},
|
||||
"K5_progressiveDisclosure": {
|
||||
"namedFileLinks": 0,
|
||||
"folderRefs": 10,
|
||||
"totalRefFiles": 153,
|
||||
"namedRatio": 0,
|
||||
"pass": false,
|
||||
"sampleNamed": []
|
||||
},
|
||||
"K6_routingTable": {
|
||||
"namedStartFiles": 0,
|
||||
"pass": false
|
||||
},
|
||||
"refCountConsistency": {
|
||||
"consistent": false,
|
||||
"mismatches": [
|
||||
{
|
||||
"folder": "agent-orchestration",
|
||||
"cited": 20,
|
||||
"actual": 24
|
||||
},
|
||||
{
|
||||
"folder": "agent-orchestration",
|
||||
"citedMultiple": [
|
||||
24,
|
||||
20
|
||||
],
|
||||
"actual": 24
|
||||
}
|
||||
]
|
||||
},
|
||||
"K9_timeSensitiveHints": {
|
||||
"timeSensitiveTokenHits": 3,
|
||||
"sample": [
|
||||
"GA",
|
||||
"preview"
|
||||
]
|
||||
}
|
||||
},
|
||||
"judgeInputs": {
|
||||
"description": "Deep technical guidance for building AI solutions in the Microsoft stack — RAG architecture, multi-agent orchestration, Azure AI Services, data engineering with Fabric, MLOps/GenAIOps, multimodal AI, API Management for AI. Triggers on: \"RAG architecture on Azure\", \"multi-agent orchestration pattern\", \"MLOps for generative AI\", \"Azure AI Search\", \"Semantic Kernel agent\", \"Fabric data pipeline\".",
|
||||
"bodyLines": 150,
|
||||
"refFileSample": [
|
||||
"skills/ms-ai-engineering/references/agent-orchestration/agent-365-governance-and-deployment.md",
|
||||
"skills/ms-ai-engineering/references/agent-orchestration/agent-autonomy-and-control-governance.md",
|
||||
"skills/ms-ai-engineering/references/agent-orchestration/agent-compliance-and-audit-trails.md",
|
||||
"skills/ms-ai-engineering/references/agent-orchestration/agent-cost-optimization-strategies.md",
|
||||
"skills/ms-ai-engineering/references/agent-orchestration/agent-ecosystem-and-plugin-marketplace.md",
|
||||
"skills/ms-ai-engineering/references/agent-orchestration/agent-evaluation-testing-frameworks.md",
|
||||
"skills/ms-ai-engineering/references/agent-orchestration/agent-feedback-and-learning-loops.md",
|
||||
"skills/ms-ai-engineering/references/agent-orchestration/agent-latency-optimization.md"
|
||||
]
|
||||
},
|
||||
"judge": {
|
||||
"K1_triggerPrecision": {
|
||||
"provisional": true,
|
||||
"precision": 0.95,
|
||||
"notes": "19/20. Miss: 'Semantic Kernel agent governance' overlaps security/governance because description lists 'governance' under agent-orchestration. Operator must curate."
|
||||
},
|
||||
"K4_noDuplication": {
|
||||
"score": 4,
|
||||
"pass": true,
|
||||
"evidence": "Mostly router. Two inline tables (RAG-vs-finetuning; MLOps test-types w/ Ragas/red-teaming) restate comparison content also in ref files — summary-level, not verbatim."
|
||||
},
|
||||
"K7_imperativeStyle": {
|
||||
"ratio": 0.9,
|
||||
"pass": true,
|
||||
"notes": "9/10; section intros are descriptive prose by design."
|
||||
},
|
||||
"K8_sourceCitation": {
|
||||
"ratio": 1,
|
||||
"pass": true,
|
||||
"notes": "5/5 dated headers; schema drift ('Dato:' vs 'Last updated'); no source-URL in headers."
|
||||
},
|
||||
"K9_noTimeSensitive": {
|
||||
"pass": false,
|
||||
"findings": [
|
||||
"Line 119: 'Foundry Agent Service GA' — explicit GA-status claim in body",
|
||||
"GPT-4o/Whisper/text-embedding-3/Florence — version-pinned product names in body",
|
||||
"Line 48: 'Agent Framework (erstatter Semantic Kernel Agents)' — lifecycle/transition claim",
|
||||
"A2A/CUA/Foundry Workflows — era-bound feature names"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ms-ai-governance",
|
||||
"skillMd": "skills/ms-ai-governance/SKILL.md",
|
||||
"refFilesActual": 78,
|
||||
"refCountsPerFolder": {
|
||||
"monitoring-observability": 18,
|
||||
"norwegian-public-sector-governance": 30,
|
||||
"responsible-ai": 30
|
||||
},
|
||||
"deterministic": {
|
||||
"K2_descriptionFormat": {
|
||||
"quotedPhrases": 6,
|
||||
"useWhenForm": true,
|
||||
"imperativeStart": false,
|
||||
"pass": true
|
||||
},
|
||||
"K3_bodyLength": {
|
||||
"bodyLines": 299,
|
||||
"pass": true
|
||||
},
|
||||
"K5_progressiveDisclosure": {
|
||||
"namedFileLinks": 24,
|
||||
"folderRefs": 5,
|
||||
"totalRefFiles": 78,
|
||||
"namedRatio": 0.3077,
|
||||
"pass": true,
|
||||
"sampleNamed": [
|
||||
"references/norwegian-public-sector-governance/utredningsinstruksen-ai-methodology.md",
|
||||
"references/norwegian-public-sector-governance/forvaltningsloven-ai-decisions.md",
|
||||
"references/norwegian-public-sector-governance/samfunnsokonomisk-analyse-nnv.md",
|
||||
"references/norwegian-public-sector-governance/gevinstrealisering-dfo-methodology.md",
|
||||
"references/norwegian-public-sector-governance/anskaffelser-ai-procurement-framework.md"
|
||||
]
|
||||
},
|
||||
"K6_routingTable": {
|
||||
"namedStartFiles": 24,
|
||||
"pass": true
|
||||
},
|
||||
"refCountConsistency": {
|
||||
"consistent": false,
|
||||
"mismatches": [
|
||||
{
|
||||
"folder": "norwegian-public-sector-governance",
|
||||
"cited": 29,
|
||||
"actual": 30
|
||||
}
|
||||
]
|
||||
},
|
||||
"K9_timeSensitiveHints": {
|
||||
"timeSensitiveTokenHits": 1,
|
||||
"sample": [
|
||||
"2024"
|
||||
]
|
||||
}
|
||||
},
|
||||
"judgeInputs": {
|
||||
"description": "Norwegian public sector AI compliance, utredningsinstruksen for AI, EU AI Act risk classification, DPIA for AI systems, Digdir architecture principles, responsible AI governance, monitoring and observability. Triggers on: \"Norwegian public sector AI compliance\", \"AI Act risk classification\", \"DPIA for AI system\", \"Digdir architecture principles\", \"ansvarlig AI i offentlig sektor\", \"Forvaltningsloven AI\".",
|
||||
"bodyLines": 299,
|
||||
"refFileSample": [
|
||||
"skills/ms-ai-governance/references/monitoring-observability/alerting-strategies-escalation.md",
|
||||
"skills/ms-ai-governance/references/monitoring-observability/anomaly-detection-ai-systems.md",
|
||||
"skills/ms-ai-governance/references/monitoring-observability/application-insights-llm-monitoring.md",
|
||||
"skills/ms-ai-governance/references/monitoring-observability/azure-monitor-setup-ai-workloads.md",
|
||||
"skills/ms-ai-governance/references/monitoring-observability/compliance-monitoring-ai-governance.md",
|
||||
"skills/ms-ai-governance/references/monitoring-observability/cost-monitoring-cost-attribution.md",
|
||||
"skills/ms-ai-governance/references/monitoring-observability/custom-dashboards-ai-operations.md",
|
||||
"skills/ms-ai-governance/references/monitoring-observability/data-residency-audit-monitoring.md"
|
||||
]
|
||||
},
|
||||
"judge": {
|
||||
"K1_triggerPrecision": {
|
||||
"provisional": true,
|
||||
"precision": 0.95,
|
||||
"notes": "19/20. Miss: Schrems II / data-transfer — body section 2.3 covers it but description has NO Schrems/dataoverføring keyword. Add trigger phrase to description."
|
||||
},
|
||||
"K4_noDuplication": {
|
||||
"score": 3,
|
||||
"pass": false,
|
||||
"evidence": "Moderate duplication: §6.1 DPIA risk-factor tree (incl. >=2-faktorer threshold) duplicates dpia-norwegian-methodology-ai.md; §6.2 + §2.1 AI Act taxonomy duplicates ai-act-classification-methodology.md; §1.2 Digdir 7-principle table restates per-principle files. Regulatory update must be applied twice."
|
||||
},
|
||||
"K7_imperativeStyle": {
|
||||
"ratio": 1,
|
||||
"pass": true,
|
||||
"notes": "10/10 imperative."
|
||||
},
|
||||
"K8_sourceCitation": {
|
||||
"ratio": 1,
|
||||
"pass": true,
|
||||
"notes": "5/5 dated headers + Status + inline source citations (Lovdata/NSM/europalov). BONUS DEFECT: SKILL.md line 191 references 'drift-detection-automated-retraining.md' which does NOT exist (actual: model-performance-drift-detection.md) — broken ref path."
|
||||
},
|
||||
"K9_noTimeSensitive": {
|
||||
"pass": false,
|
||||
"findings": [
|
||||
"Line 121: 'Microsoft EU Data Boundary ... Azure OpenAI (Sweden Central, West Europe)' — volatile region/availability claim in body (most genuine finding)",
|
||||
"Lines 81/121: 'Regulation 2024/1689' + 'Schrems II (C-311/18)' — legal identifiers (stable, borderline)",
|
||||
"§2.1/§6.2 AI Act obligation status presented without Digital Omnibus caveat — may date"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ms-ai-infrastructure",
|
||||
"skillMd": "skills/ms-ai-infrastructure/SKILL.md",
|
||||
"refFilesActual": 34,
|
||||
"refCountsPerFolder": {
|
||||
"bcdr": 16,
|
||||
"hybrid-edge": 18
|
||||
},
|
||||
"deterministic": {
|
||||
"K2_descriptionFormat": {
|
||||
"quotedPhrases": 5,
|
||||
"useWhenForm": true,
|
||||
"imperativeStart": false,
|
||||
"pass": true
|
||||
},
|
||||
"K3_bodyLength": {
|
||||
"bodyLines": 290,
|
||||
"pass": true
|
||||
},
|
||||
"K5_progressiveDisclosure": {
|
||||
"namedFileLinks": 33,
|
||||
"folderRefs": 5,
|
||||
"totalRefFiles": 34,
|
||||
"namedRatio": 0.9706,
|
||||
"pass": true,
|
||||
"sampleNamed": [
|
||||
"references/bcdr/multi-region-azure-openai-deployment.md",
|
||||
"references/bcdr/rto-rpo-planning-ai-services.md",
|
||||
"references/bcdr/backup-recovery-strategies-ai-workloads.md",
|
||||
"references/bcdr/failover-testing-ai-services.md",
|
||||
"references/bcdr/chaos-engineering-ai-systems.md"
|
||||
]
|
||||
},
|
||||
"K6_routingTable": {
|
||||
"namedStartFiles": 33,
|
||||
"pass": true
|
||||
},
|
||||
"refCountConsistency": {
|
||||
"consistent": true,
|
||||
"mismatches": []
|
||||
},
|
||||
"K9_timeSensitiveHints": {
|
||||
"timeSensitiveTokenHits": 7,
|
||||
"sample": [
|
||||
"preview",
|
||||
"GA",
|
||||
"v2"
|
||||
]
|
||||
}
|
||||
},
|
||||
"judgeInputs": {
|
||||
"description": "Disaster recovery for AI workloads, multi-region Azure AI deployment, hybrid or edge AI architecture, sovereign cloud for Norway, offline-first AI patterns, AI infrastructure resilience. Covers BCDR, Azure Arc for AI, ONNX Runtime edge deployment, disconnected scenarios, Norwegian data sovereignty. Triggers on: \"disaster recovery for AI workloads\", \"edge AI deployment\", \"sovereign cloud AI\", \"Azure Arc for AI\", \"BCDR for AI\".",
|
||||
"bodyLines": 290,
|
||||
"refFileSample": [
|
||||
"skills/ms-ai-infrastructure/references/bcdr/ai-foundry-disaster-recovery-planning.md",
|
||||
"skills/ms-ai-infrastructure/references/bcdr/backup-recovery-strategies-ai-workloads.md",
|
||||
"skills/ms-ai-infrastructure/references/bcdr/capacity-planning-dr-configurations.md",
|
||||
"skills/ms-ai-infrastructure/references/bcdr/chaos-engineering-ai-systems.md",
|
||||
"skills/ms-ai-infrastructure/references/bcdr/compliance-requirements-bcdr.md",
|
||||
"skills/ms-ai-infrastructure/references/bcdr/cost-analysis-dr-configurations.md",
|
||||
"skills/ms-ai-infrastructure/references/bcdr/data-replication-patterns-ai.md",
|
||||
"skills/ms-ai-infrastructure/references/bcdr/failover-testing-ai-services.md"
|
||||
]
|
||||
},
|
||||
"judge": {
|
||||
"K1_triggerPrecision": {
|
||||
"provisional": true,
|
||||
"precision": 1,
|
||||
"notes": "20/20. Tightly scoped + named trigger phrases, low false-positive risk. Borderline (hybrid RAG, multi-region cost) need real-corpus validation. Operator must curate."
|
||||
},
|
||||
"K4_noDuplication": {
|
||||
"score": 4,
|
||||
"pass": true,
|
||||
"evidence": "Largely summary/routing w/ '> Ref:' deferral. Minor: RTO/RPO table + SLA table put specific values in body that also live in ref files."
|
||||
},
|
||||
"K7_imperativeStyle": {
|
||||
"ratio": 1,
|
||||
"pass": true,
|
||||
"notes": "10/10 imperative (concentrated in per-section directives)."
|
||||
},
|
||||
"K8_sourceCitation": {
|
||||
"ratio": 1,
|
||||
"pass": true,
|
||||
"notes": "5/5 'Last updated: 2026-02' + Status. NO source-URL in headers; identical batch date reads as generation timestamp, not per-file verification."
|
||||
},
|
||||
"K9_noTimeSensitive": {
|
||||
"pass": false,
|
||||
"findings": [
|
||||
"Lines 93-98: SLA table w/ hardcoded percentages (99.9/99.999/99.95%) + 'Standard v2'",
|
||||
"Lines 149/158/184/186/249: Phi-3/Phi-4 model versions + param counts (3.8B/14B) in body",
|
||||
"Line 141: 'Azure Local (tidl. Azure Stack HCI)' rename note",
|
||||
"Line 87: hardcoded 'peak + 30% buffer'"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ms-ai-security",
|
||||
"skillMd": "skills/ms-ai-security/SKILL.md",
|
||||
"refFilesActual": 62,
|
||||
"refCountsPerFolder": {
|
||||
"ai-security-engineering": 22,
|
||||
"cost-optimization": 22,
|
||||
"performance-scalability": 18
|
||||
},
|
||||
"deterministic": {
|
||||
"K2_descriptionFormat": {
|
||||
"quotedPhrases": 6,
|
||||
"useWhenForm": true,
|
||||
"imperativeStart": false,
|
||||
"pass": true
|
||||
},
|
||||
"K3_bodyLength": {
|
||||
"bodyLines": 212,
|
||||
"pass": true
|
||||
},
|
||||
"K5_progressiveDisclosure": {
|
||||
"namedFileLinks": 10,
|
||||
"folderRefs": 6,
|
||||
"totalRefFiles": 62,
|
||||
"namedRatio": 0.1613,
|
||||
"pass": false,
|
||||
"sampleNamed": [
|
||||
"references/ai-security-engineering/security-scoring-rubrics-6x5.md",
|
||||
"references/ai-security-engineering/ai-security-scoring-framework.md",
|
||||
"references/ai-security-engineering/secure-model-deployment-hardening.md",
|
||||
"references/ai-security-engineering/zero-trust-ai-services.md",
|
||||
"references/cost-optimization/deterministic-cost-calculation-model.md"
|
||||
]
|
||||
},
|
||||
"K6_routingTable": {
|
||||
"namedStartFiles": 10,
|
||||
"pass": true
|
||||
},
|
||||
"refCountConsistency": {
|
||||
"consistent": true,
|
||||
"mismatches": []
|
||||
},
|
||||
"K9_timeSensitiveHints": {
|
||||
"timeSensitiveTokenHits": 3,
|
||||
"sample": [
|
||||
"2025",
|
||||
"GA",
|
||||
"Preview"
|
||||
]
|
||||
}
|
||||
},
|
||||
"judgeInputs": {
|
||||
"description": "Security assessment, cost estimation, OWASP LLM Top 10 mitigations, performance optimization for AI on Microsoft stack. Deterministic 6x5 security scoring, P10/P50/P90 cost confidence intervals, FinOps practices. Triggers on: \"security assessment for AI\", \"AI threat modeling\", \"cost estimation for Azure AI\", \"FinOps for AI workloads\", \"OWASP LLM\", \"kostnadsestimat for AI-løsning\".",
|
||||
"bodyLines": 212,
|
||||
"refFileSample": [
|
||||
"skills/ms-ai-security/references/ai-security-engineering/adversarial-input-robustness-testing.md",
|
||||
"skills/ms-ai-security/references/ai-security-engineering/ai-incident-response-procedures.md",
|
||||
"skills/ms-ai-security/references/ai-security-engineering/ai-prompt-shield-network.md",
|
||||
"skills/ms-ai-security/references/ai-security-engineering/ai-red-team-operations-practical.md",
|
||||
"skills/ms-ai-security/references/ai-security-engineering/ai-security-scoring-framework.md",
|
||||
"skills/ms-ai-security/references/ai-security-engineering/ai-threat-modeling-stride.md",
|
||||
"skills/ms-ai-security/references/ai-security-engineering/content-safety-filter-calibration.md",
|
||||
"skills/ms-ai-security/references/ai-security-engineering/data-leakage-prevention-ai.md"
|
||||
]
|
||||
},
|
||||
"judge": {
|
||||
"K1_triggerPrecision": {
|
||||
"provisional": true,
|
||||
"precision": 0.95,
|
||||
"notes": "19/20. Clean sibling separation (AI Act/DPIA/platform/RAG/BCDR not triggered). One ambiguity: model feature-comparison could be pulled by broad 'performance optimization for AI'. Operator must curate."
|
||||
},
|
||||
"K4_noDuplication": {
|
||||
"score": 3,
|
||||
"pass": false,
|
||||
"evidence": "DUPLICATION-WITH-CONTRADICTION: body weighting table (L52-59 Standard: Identity 20/Network 15/Data 20/Content 20/Compliance 15/Monitoring 10) CONTRADICTS canonical rubric security-scoring-rubrics-6x5.md (Compliance 25/Data 20/Identity 20/Content 15/Network 10/Monitoring 10). Body scoring rule (weighted sum) also diverges from rubric (Ja-checkpoint count). Real correctness bug."
|
||||
},
|
||||
"K7_imperativeStyle": {
|
||||
"ratio": 1,
|
||||
"pass": true,
|
||||
"notes": "10/10 imperative."
|
||||
},
|
||||
"K8_sourceCitation": {
|
||||
"ratio": 1,
|
||||
"pass": true,
|
||||
"notes": "5/5 dated headers; mostly no header source-URL (URLs in body cost-register rows)."
|
||||
},
|
||||
"K9_noTimeSensitive": {
|
||||
"pass": false,
|
||||
"findings": [
|
||||
"Line 73: 'OWASP LLM Top 10 (2025)' — dated standard version",
|
||||
"Line 92: 'Defender ... GA for AI applications, Preview for AI agents' + 'ikke i Azure Government' — GA/preview + region status in body",
|
||||
"Line 138: 'GPT-4o mini vs GPT-4o' — model versions",
|
||||
"Lines 149-153: hardcoded perf/price figures (20-50ms, 5-10x, 50%/80%, Batch 50% @ 24h SLA)"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
43
scripts/kb-eval/data/judge-results.json
Normal file
43
scripts/kb-eval/data/judge-results.json
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
{
|
||||
"_meta": {
|
||||
"rubric": "scripts/kb-eval/judge-prompt.md",
|
||||
"judge_model": "opus",
|
||||
"method": "5 parallel adversarial LLM-judges, one per skill",
|
||||
"note": "K1 precision is PROVISIONAL — operator must curate the final 20 trigger-prompts per skill before K1 is authoritative."
|
||||
},
|
||||
"ms-ai-advisor": {
|
||||
"K1_triggerPrecision": { "provisional": true, "precision": 1.0, "notes": "20/20 from description. BORDERLINE: cost/diagram/DPIA/AI-Act prompts are MS-AI-adjacent; broad 'Microsoft AI architecture' phrase risks over-triggering vs sibling commands. Operator must curate + stress-test sibling overlap." },
|
||||
"K4_noDuplication": { "score": 5, "pass": true, "evidence": "Body = persona + 7-phase workflow + ref-index; no ref detail reproduced. Only internal MCP-table redundancy (SKILL-internal, not SKILL<->ref)." },
|
||||
"K7_imperativeStyle": { "ratio": 1.0, "pass": true, "notes": "10/10 sampled instruction sentences imperative." },
|
||||
"K8_sourceCitation": { "ratio": 0.8, "pass": true, "notes": "AT THRESHOLD: architecture/decision-trees.md lacks dated header (footer source only). Add dated header to harden margin." },
|
||||
"K9_noTimeSensitive": { "pass": true, "findings": ["Only meta-instructions (preview/GA as dynamic-to-verify) + stable identifiers (M365 SKUs, MADR v3.0). No stale-able product claim in body."] }
|
||||
},
|
||||
"ms-ai-engineering": {
|
||||
"K1_triggerPrecision": { "provisional": true, "precision": 0.95, "notes": "19/20. Miss: 'Semantic Kernel agent governance' overlaps security/governance because description lists 'governance' under agent-orchestration. Operator must curate." },
|
||||
"K4_noDuplication": { "score": 4, "pass": true, "evidence": "Mostly router. Two inline tables (RAG-vs-finetuning; MLOps test-types w/ Ragas/red-teaming) restate comparison content also in ref files — summary-level, not verbatim." },
|
||||
"K7_imperativeStyle": { "ratio": 0.9, "pass": true, "notes": "9/10; section intros are descriptive prose by design." },
|
||||
"K8_sourceCitation": { "ratio": 1.0, "pass": true, "notes": "5/5 dated headers; schema drift ('Dato:' vs 'Last updated'); no source-URL in headers." },
|
||||
"K9_noTimeSensitive": { "pass": false, "findings": ["Line 119: 'Foundry Agent Service GA' — explicit GA-status claim in body", "GPT-4o/Whisper/text-embedding-3/Florence — version-pinned product names in body", "Line 48: 'Agent Framework (erstatter Semantic Kernel Agents)' — lifecycle/transition claim", "A2A/CUA/Foundry Workflows — era-bound feature names"] }
|
||||
},
|
||||
"ms-ai-governance": {
|
||||
"K1_triggerPrecision": { "provisional": true, "precision": 0.95, "notes": "19/20. Miss: Schrems II / data-transfer — body section 2.3 covers it but description has NO Schrems/dataoverføring keyword. Add trigger phrase to description." },
|
||||
"K4_noDuplication": { "score": 3, "pass": false, "evidence": "Moderate duplication: §6.1 DPIA risk-factor tree (incl. >=2-faktorer threshold) duplicates dpia-norwegian-methodology-ai.md; §6.2 + §2.1 AI Act taxonomy duplicates ai-act-classification-methodology.md; §1.2 Digdir 7-principle table restates per-principle files. Regulatory update must be applied twice." },
|
||||
"K7_imperativeStyle": { "ratio": 1.0, "pass": true, "notes": "10/10 imperative." },
|
||||
"K8_sourceCitation": { "ratio": 1.0, "pass": true, "notes": "5/5 dated headers + Status + inline source citations (Lovdata/NSM/europalov). BONUS DEFECT: SKILL.md line 191 references 'drift-detection-automated-retraining.md' which does NOT exist (actual: model-performance-drift-detection.md) — broken ref path." },
|
||||
"K9_noTimeSensitive": { "pass": false, "findings": ["Line 121: 'Microsoft EU Data Boundary ... Azure OpenAI (Sweden Central, West Europe)' — volatile region/availability claim in body (most genuine finding)", "Lines 81/121: 'Regulation 2024/1689' + 'Schrems II (C-311/18)' — legal identifiers (stable, borderline)", "§2.1/§6.2 AI Act obligation status presented without Digital Omnibus caveat — may date"] }
|
||||
},
|
||||
"ms-ai-infrastructure": {
|
||||
"K1_triggerPrecision": { "provisional": true, "precision": 1.0, "notes": "20/20. Tightly scoped + named trigger phrases, low false-positive risk. Borderline (hybrid RAG, multi-region cost) need real-corpus validation. Operator must curate." },
|
||||
"K4_noDuplication": { "score": 4, "pass": true, "evidence": "Largely summary/routing w/ '> Ref:' deferral. Minor: RTO/RPO table + SLA table put specific values in body that also live in ref files." },
|
||||
"K7_imperativeStyle": { "ratio": 1.0, "pass": true, "notes": "10/10 imperative (concentrated in per-section directives)." },
|
||||
"K8_sourceCitation": { "ratio": 1.0, "pass": true, "notes": "5/5 'Last updated: 2026-02' + Status. NO source-URL in headers; identical batch date reads as generation timestamp, not per-file verification." },
|
||||
"K9_noTimeSensitive": { "pass": false, "findings": ["Lines 93-98: SLA table w/ hardcoded percentages (99.9/99.999/99.95%) + 'Standard v2'", "Lines 149/158/184/186/249: Phi-3/Phi-4 model versions + param counts (3.8B/14B) in body", "Line 141: 'Azure Local (tidl. Azure Stack HCI)' rename note", "Line 87: hardcoded 'peak + 30% buffer'"] }
|
||||
},
|
||||
"ms-ai-security": {
|
||||
"K1_triggerPrecision": { "provisional": true, "precision": 0.95, "notes": "19/20. Clean sibling separation (AI Act/DPIA/platform/RAG/BCDR not triggered). One ambiguity: model feature-comparison could be pulled by broad 'performance optimization for AI'. Operator must curate." },
|
||||
"K4_noDuplication": { "score": 3, "pass": false, "evidence": "DUPLICATION-WITH-CONTRADICTION: body weighting table (L52-59 Standard: Identity 20/Network 15/Data 20/Content 20/Compliance 15/Monitoring 10) CONTRADICTS canonical rubric security-scoring-rubrics-6x5.md (Compliance 25/Data 20/Identity 20/Content 15/Network 10/Monitoring 10). Body scoring rule (weighted sum) also diverges from rubric (Ja-checkpoint count). Real correctness bug." },
|
||||
"K7_imperativeStyle": { "ratio": 1.0, "pass": true, "notes": "10/10 imperative." },
|
||||
"K8_sourceCitation": { "ratio": 1.0, "pass": true, "notes": "5/5 dated headers; mostly no header source-URL (URLs in body cost-register rows)." },
|
||||
"K9_noTimeSensitive": { "pass": false, "findings": ["Line 73: 'OWASP LLM Top 10 (2025)' — dated standard version", "Line 92: 'Defender ... GA for AI applications, Preview for AI agents' + 'ikke i Azure Government' — GA/preview + region status in body", "Line 138: 'GPT-4o mini vs GPT-4o' — model versions", "Lines 149-153: hardcoded perf/price figures (20-50ms, 5-10x, 50%/80%, Batch 50% @ 24h SLA)"] }
|
||||
}
|
||||
}
|
||||
257
scripts/kb-eval/eval.mjs
Normal file
257
scripts/kb-eval/eval.mjs
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
#!/usr/bin/env node
|
||||
// eval.mjs — Skill quality baseline (rubrikk K1–K9) for ms-ai-architect.
|
||||
//
|
||||
// READ-ONLY by default. Computes the deterministic criteria (K2/K3/K5/K6 +
|
||||
// reference-count consistency) for every skill and extracts the inputs an LLM
|
||||
// judge needs for the semantic criteria (K1/K4/K7/K8/K9). The baseline file is
|
||||
// written ONLY with --write (operator gate), per the redesign spec.
|
||||
//
|
||||
// Usage:
|
||||
// node scripts/kb-eval/eval.mjs # human-readable summary
|
||||
// node scripts/kb-eval/eval.mjs --json # machine output (pure JSON to stdout)
|
||||
// node scripts/kb-eval/eval.mjs --write # also persist data/eval-baseline.json
|
||||
//
|
||||
// Zero dependencies. Reuses kb-update/lib/atomic-write.mjs for the gated write.
|
||||
|
||||
import { readdirSync, readFileSync, existsSync, mkdirSync } from 'node:fs';
|
||||
import { join, dirname, relative } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PLUGIN_ROOT = join(__dirname, '..', '..');
|
||||
const SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
|
||||
const OUT_DIR = join(__dirname, 'data');
|
||||
const OUT_FILE = join(OUT_DIR, 'eval-baseline.json');
|
||||
|
||||
// Thresholds — see spec "Rubrikk K1–K9".
|
||||
const K3_MAX_BODY_LINES = 500;
|
||||
const K5_MIN_NAMED_RATIO = 0.2;
|
||||
|
||||
/** Recursively list .md files under dir. */
|
||||
function listMarkdown(dir) {
|
||||
const out = [];
|
||||
if (!existsSync(dir)) return out;
|
||||
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
||||
const p = join(dir, e.name);
|
||||
if (e.isDirectory()) out.push(...listMarkdown(p));
|
||||
else if (e.isFile() && e.name.endsWith('.md')) out.push(p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Count .md files per immediate subfolder of references/. */
|
||||
function refCountsPerFolder(refDir) {
|
||||
const counts = {};
|
||||
if (!existsSync(refDir)) return counts;
|
||||
for (const e of readdirSync(refDir, { withFileTypes: true })) {
|
||||
if (e.isDirectory()) counts[e.name] = listMarkdown(join(refDir, e.name)).length;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
/** Split YAML frontmatter from body. */
|
||||
export function splitFrontmatter(content) {
|
||||
if (!content.startsWith('---')) return { frontmatter: '', body: content };
|
||||
const close = content.indexOf('\n---', 3);
|
||||
if (close === -1) return { frontmatter: '', body: content };
|
||||
const bodyStart = content.indexOf('\n', close + 1);
|
||||
return {
|
||||
frontmatter: content.slice(0, bodyStart === -1 ? content.length : bodyStart + 1),
|
||||
body: bodyStart === -1 ? '' : content.slice(bodyStart + 1),
|
||||
};
|
||||
}
|
||||
|
||||
/** Extract the (possibly folded) description value from frontmatter. */
|
||||
export function extractDescription(frontmatter) {
|
||||
const lines = frontmatter.split('\n');
|
||||
const out = [];
|
||||
let collecting = false;
|
||||
for (const line of lines) {
|
||||
if (/^description:/.test(line)) {
|
||||
collecting = true;
|
||||
out.push(line.replace(/^description:\s*>?-?\s*/, ''));
|
||||
continue;
|
||||
}
|
||||
if (collecting) {
|
||||
if (/^[A-Za-z0-9_-]+:\s/.test(line) || line.trim() === '---' || line.trim() === '') {
|
||||
if (line.trim() === '') continue; // folded blocks keep going across blanks
|
||||
break;
|
||||
}
|
||||
out.push(line.trim());
|
||||
}
|
||||
}
|
||||
return out.join(' ').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
/** K2 — description format: third-person/use-when OR >=3 quoted user phrases. */
|
||||
export function checkK2(description) {
|
||||
const quoted = (description.match(/"[^"]+"/g) || []).length;
|
||||
const useWhenForm = /should be used when|use this skill|triggers on/i.test(description);
|
||||
const imperativeStart = /^(use|bruk|run|kjør)\b/i.test(description);
|
||||
const pass = useWhenForm || quoted >= 3;
|
||||
return { quotedPhrases: quoted, useWhenForm, imperativeStart, pass };
|
||||
}
|
||||
|
||||
/** K3 — SKILL.md body length. */
|
||||
export function checkK3(body) {
|
||||
const lines = body.replace(/\n+$/, '').split('\n').length;
|
||||
return { bodyLines: lines, pass: lines <= K3_MAX_BODY_LINES };
|
||||
}
|
||||
|
||||
/** K5/K6 — progressive disclosure: named .md links vs folder-only references. */
|
||||
export function checkK5K6(body, totalRefFiles) {
|
||||
const named = [...new Set((body.match(/references\/[A-Za-z0-9_\-/]+\.md/g) || []))];
|
||||
// folder references like `references/foo/` not immediately followed by a filename
|
||||
const folderRefs = [...new Set((body.match(/references\/[A-Za-z0-9_-]+\/(?![A-Za-z0-9_-]+\.md)/g) || []))];
|
||||
const namedRatio = totalRefFiles > 0 ? named.length / totalRefFiles : 0;
|
||||
return {
|
||||
K5: {
|
||||
namedFileLinks: named.length,
|
||||
folderRefs: folderRefs.length,
|
||||
totalRefFiles,
|
||||
namedRatio: Number(namedRatio.toFixed(4)),
|
||||
pass: namedRatio >= K5_MIN_NAMED_RATIO,
|
||||
sampleNamed: named.slice(0, 5),
|
||||
},
|
||||
K6: {
|
||||
// routing table = at least one named start file the model can Read directly
|
||||
namedStartFiles: named.length,
|
||||
pass: named.length >= 1,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Reference-count consistency: cited per-folder counts vs actual on disk. */
|
||||
export function checkRefConsistency(body, actualPerFolder) {
|
||||
const mismatches = [];
|
||||
const seen = {};
|
||||
// prose form: references/<folder>/ (N files) and `references/<folder>/` (N files)
|
||||
const proseRe = /references\/([a-z0-9-]+)\/`?\s*\((\d+)\s*files?\)/gi;
|
||||
// table form: `references/<folder>/` | N
|
||||
const tableRe = /`references\/([a-z0-9-]+)\/`\s*\|\s*(\d+)/gi;
|
||||
for (const re of [proseRe, tableRe]) {
|
||||
let m;
|
||||
while ((m = re.exec(body)) !== null) {
|
||||
const folder = m[1];
|
||||
const cited = Number(m[2]);
|
||||
seen[folder] = seen[folder] || new Set();
|
||||
seen[folder].add(cited);
|
||||
const actual = actualPerFolder[folder];
|
||||
if (actual !== undefined && cited !== actual) {
|
||||
mismatches.push({ folder, cited, actual });
|
||||
}
|
||||
}
|
||||
}
|
||||
// also flag folders cited with two different numbers
|
||||
for (const [folder, set] of Object.entries(seen)) {
|
||||
if (set.size > 1) mismatches.push({ folder, citedMultiple: [...set], actual: actualPerFolder[folder] ?? null });
|
||||
}
|
||||
return { consistent: mismatches.length === 0, mismatches };
|
||||
}
|
||||
|
||||
/** Light deterministic pre-scan for K9 (time-sensitive tokens in body). */
|
||||
function scanK9Hints(body) {
|
||||
const tokens = body.match(/\b(GA|preview|deprecat\w*|retir\w*|20\d\d|v\d+(?:\.\d+)?)\b/gi) || [];
|
||||
return { timeSensitiveTokenHits: tokens.length, sample: [...new Set(tokens)].slice(0, 12) };
|
||||
}
|
||||
|
||||
function evalSkill(skillName) {
|
||||
const skillDir = join(SKILLS_DIR, skillName);
|
||||
const skillMd = join(skillDir, 'SKILL.md');
|
||||
if (!existsSync(skillMd)) return null;
|
||||
const content = readFileSync(skillMd, 'utf8');
|
||||
const { frontmatter, body } = splitFrontmatter(content);
|
||||
const description = extractDescription(frontmatter);
|
||||
const refDir = join(skillDir, 'references');
|
||||
const refFiles = listMarkdown(refDir);
|
||||
const perFolder = refCountsPerFolder(refDir);
|
||||
|
||||
const K2 = checkK2(description);
|
||||
const K3 = checkK3(body);
|
||||
const { K5, K6 } = checkK5K6(body, refFiles.length);
|
||||
const refConsistency = checkRefConsistency(body, perFolder);
|
||||
const K9hints = scanK9Hints(body);
|
||||
|
||||
return {
|
||||
name: skillName,
|
||||
skillMd: relative(PLUGIN_ROOT, skillMd),
|
||||
refFilesActual: refFiles.length,
|
||||
refCountsPerFolder: perFolder,
|
||||
deterministic: {
|
||||
K2_descriptionFormat: K2,
|
||||
K3_bodyLength: K3,
|
||||
K5_progressiveDisclosure: K5,
|
||||
K6_routingTable: K6,
|
||||
refCountConsistency: refConsistency,
|
||||
K9_timeSensitiveHints: K9hints,
|
||||
},
|
||||
// Pointers for the LLM-judge pass (K1/K4/K7/K8/K9). The judge reads the files
|
||||
// directly; we only carry the description + sampled ref files here.
|
||||
judgeInputs: {
|
||||
description,
|
||||
bodyLines: K3.bodyLines,
|
||||
refFileSample: refFiles.slice(0, 8).map((f) => relative(PLUGIN_ROOT, f)),
|
||||
},
|
||||
// Filled in by the operator-gated LLM-judge merge step. Null = not yet judged.
|
||||
judge: null,
|
||||
};
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const jsonOut = args.includes('--json');
|
||||
const doWrite = args.includes('--write');
|
||||
|
||||
const skillNames = readdirSync(SKILLS_DIR, { withFileTypes: true })
|
||||
.filter((e) => e.isDirectory() && existsSync(join(SKILLS_DIR, e.name, 'SKILL.md')))
|
||||
.map((e) => e.name)
|
||||
.sort();
|
||||
|
||||
const skills = skillNames.map(evalSkill).filter(Boolean);
|
||||
|
||||
// Merge operator-gated LLM-judge results (K1/K4/K7/K8/K9) if present.
|
||||
const judgeFile = join(OUT_DIR, 'judge-results.json');
|
||||
if (existsSync(judgeFile)) {
|
||||
const jr = JSON.parse(readFileSync(judgeFile, 'utf8'));
|
||||
for (const s of skills) if (jr[s.name]) s.judge = jr[s.name];
|
||||
}
|
||||
|
||||
const report = {
|
||||
rubric: 'K1-K9',
|
||||
note: 'Deterministic: K2,K3,K5,K6,refCountConsistency. LLM-judge (operator-gated): K1,K4,K7,K8,K9.',
|
||||
skills,
|
||||
};
|
||||
|
||||
if (doWrite) {
|
||||
mkdirSync(OUT_DIR, { recursive: true });
|
||||
atomicWriteJson(OUT_FILE, report);
|
||||
}
|
||||
|
||||
if (jsonOut) {
|
||||
process.stdout.write(JSON.stringify(report) + '\n');
|
||||
return;
|
||||
}
|
||||
|
||||
// Human-readable summary
|
||||
console.log(`\nKB skill eval — ${skills.length} skills (deterministic pass)\n`);
|
||||
for (const s of skills) {
|
||||
const d = s.deterministic;
|
||||
const flag = (b) => (b ? 'PASS' : 'FAIL');
|
||||
console.log(`■ ${s.name} (${s.refFilesActual} ref-filer)`);
|
||||
console.log(` K2 description : ${flag(d.K2_descriptionFormat.pass)} (quoted=${d.K2_descriptionFormat.quotedPhrases}, useWhen=${d.K2_descriptionFormat.useWhenForm})`);
|
||||
console.log(` K3 body ≤500 : ${flag(d.K3_bodyLength.pass)} (${d.K3_bodyLength.bodyLines} linjer)`);
|
||||
console.log(` K5 prog.disc. : ${flag(d.K5_progressiveDisclosure.pass)} (navngitte=${d.K5_progressiveDisclosure.namedFileLinks}, mapper=${d.K5_progressiveDisclosure.folderRefs}, ratio=${d.K5_progressiveDisclosure.namedRatio})`);
|
||||
console.log(` K6 routing-tab : ${flag(d.K6_routingTable.pass)} (navngitte startfiler=${d.K6_routingTable.namedStartFiles})`);
|
||||
const rc = d.refCountConsistency;
|
||||
console.log(` ref-tall : ${flag(rc.consistent)}${rc.consistent ? '' : ' — ' + JSON.stringify(rc.mismatches)}`);
|
||||
console.log(` K9 hints : ${d.K9_timeSensitiveHints.timeSensitiveTokenHits} tid-sensitive tokens i body`);
|
||||
console.log('');
|
||||
}
|
||||
console.log(`(LLM-judge K1/K4/K7/K8/K9 kjøres som operatør-gated subagent-steg; baseline skrives med --write.)\n`);
|
||||
}
|
||||
|
||||
// Run only when invoked directly (not when imported by tests).
|
||||
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
main();
|
||||
}
|
||||
34
scripts/kb-eval/judge-prompt.md
Normal file
34
scripts/kb-eval/judge-prompt.md
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
# LLM-judge-rubrikk (K1/K4/K7/K8/K9) — skill-kvalitet, ms-ai-architect
|
||||
|
||||
Pinnet rubrikk for de semantiske eval-kriteriene som ikke kan måles deterministisk.
|
||||
Kjøres som operatør-gated subagent-steg (Opus, én dommer per skill). De deterministiske
|
||||
kriteriene (K2/K3/K5/K6 + ref-tall) måles av `eval.mjs`; denne dekker resten.
|
||||
|
||||
Placeholders: `<NAME>` = skill-navn, `<ROOT>` = plugin-rot, `<PATH>` = sti til SKILL.md.
|
||||
|
||||
---
|
||||
|
||||
Du er en skill-kvalitets-dommer (LLM-as-judge) for ms-ai-architect. Vurder skillen
|
||||
`<NAME>` mot K1/K4/K7/K8/K9. Plugin-rot: `<ROOT>`. SKILL.md: `<PATH>`. Referansefiler:
|
||||
`<ROOT>/skills/<NAME>/references/`.
|
||||
|
||||
Les SKILL.md i sin helhet. For K8: sample 5 tilfeldige referansefiler og les headeren deres.
|
||||
Vær streng og adversariell — ikke ros, ikke pynt på tall.
|
||||
|
||||
- **K1 trigger-presisjon [PROVISIONAL]:** generer 10 in-domain + 10 out-of-domain prompts for
|
||||
domenet. For hver, avgjør KUN fra `description`-feltet om skillen burde trigge. Rapporter
|
||||
provisional precision (korrekte/20). FLAGG eksplisitt at operatør må kuratere de endelige 20.
|
||||
- **K4 ingen duplisering (SKILL.md ↔ ref-filer):** score 1–5 (5 = ingen duplisering). Finnes detalj
|
||||
i SKILL.md body som dupliserer ref-filer? Gi konkret eksempel. pass = score ≥ 4.
|
||||
- **K7 imperativ/instruksjons-stil:** sample 10 instruksjonssetninger fra body; andel i
|
||||
imperativ/infinitiv. pass = ratio ≥ 0.80.
|
||||
- **K8 kildehenvisning i ref-filer:** for 5 samplede ref-filer, har header `Last updated` /
|
||||
`Verified` / kilde-URL? Rapporter andel. pass = ratio ≥ 0.80.
|
||||
- **K9 ingen tid-sensitiv info i SKILL.md body:** finnes datoer/versjoner/GA/preview-status DIREKTE
|
||||
i body (ikke i ref-filer)? List funn. pass = ingen funn.
|
||||
|
||||
RETURNER KUN dette JSON-objektet (ingen annen tekst, ingen markdown-fence):
|
||||
|
||||
```
|
||||
{"skill":"<NAME>","K1_triggerPrecision":{"provisional":true,"precision":0.0,"notes":""},"K4_noDuplication":{"score":0,"pass":false,"evidence":""},"K7_imperativeStyle":{"ratio":0.0,"pass":false,"notes":""},"K8_sourceCitation":{"ratio":0.0,"pass":false,"sampledFiles":[],"notes":""},"K9_noTimeSensitive":{"pass":false,"findings":[]}}
|
||||
```
|
||||
102
tests/kb-eval/test-eval.test.mjs
Normal file
102
tests/kb-eval/test-eval.test.mjs
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
// tests/kb-eval/test-eval.test.mjs
|
||||
// Unit tests for the deterministic scoring functions in scripts/kb-eval/eval.mjs
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
splitFrontmatter,
|
||||
extractDescription,
|
||||
checkK2,
|
||||
checkK3,
|
||||
checkK5K6,
|
||||
checkRefConsistency,
|
||||
} from '../../scripts/kb-eval/eval.mjs';
|
||||
|
||||
test('splitFrontmatter — separates frontmatter from body', () => {
|
||||
const c = '---\nname: x\ndescription: hi\n---\n# Body\nline2\n';
|
||||
const { body } = splitFrontmatter(c);
|
||||
assert.ok(body.startsWith('# Body'), `body was: ${JSON.stringify(body)}`);
|
||||
assert.ok(!body.includes('name: x'));
|
||||
});
|
||||
|
||||
test('splitFrontmatter — no frontmatter returns full content as body', () => {
|
||||
const { frontmatter, body } = splitFrontmatter('# No frontmatter\ntext');
|
||||
assert.equal(frontmatter, '');
|
||||
assert.equal(body, '# No frontmatter\ntext');
|
||||
});
|
||||
|
||||
test('extractDescription — extracts folded description value', () => {
|
||||
const fm = '---\nname: x\ndescription: >-\n Deep guidance. Triggers on: "a", "b".\nother: y\n---\n';
|
||||
const desc = extractDescription(fm);
|
||||
assert.match(desc, /Deep guidance/);
|
||||
assert.match(desc, /Triggers on/);
|
||||
assert.ok(!desc.includes('other: y'));
|
||||
});
|
||||
|
||||
test('checkK2 — >=3 quoted phrases passes', () => {
|
||||
const r = checkK2('Some guidance. "alpha" and "beta" and "gamma".');
|
||||
assert.equal(r.quotedPhrases, 3);
|
||||
assert.equal(r.pass, true);
|
||||
});
|
||||
|
||||
test('checkK2 — use-when form passes even without quotes', () => {
|
||||
const r = checkK2('This skill should be used when designing X.');
|
||||
assert.equal(r.useWhenForm, true);
|
||||
assert.equal(r.pass, true);
|
||||
});
|
||||
|
||||
test('checkK2 — plain description without quotes or use-when fails', () => {
|
||||
const r = checkK2('A plain description of the domain.');
|
||||
assert.equal(r.quotedPhrases, 0);
|
||||
assert.equal(r.useWhenForm, false);
|
||||
assert.equal(r.pass, false);
|
||||
});
|
||||
|
||||
test('checkK3 — body within 500 lines passes', () => {
|
||||
const r = checkK3('line1\nline2\nline3\n');
|
||||
assert.equal(r.bodyLines, 3);
|
||||
assert.equal(r.pass, true);
|
||||
});
|
||||
|
||||
test('checkK3 — body over 500 lines fails', () => {
|
||||
const r = checkK3('x\n'.repeat(501));
|
||||
assert.equal(r.pass, false);
|
||||
assert.ok(r.bodyLines > 500);
|
||||
});
|
||||
|
||||
test('checkK5K6 — named file links count toward progressive disclosure', () => {
|
||||
const body = 'See references/foo/bar.md and references/foo/baz.md for detail.';
|
||||
const { K5, K6 } = checkK5K6(body, 10);
|
||||
assert.equal(K5.namedFileLinks, 2);
|
||||
assert.equal(K5.namedRatio, 0.2);
|
||||
assert.equal(K5.pass, true); // 0.2 >= 0.20
|
||||
assert.equal(K6.pass, true); // >=1 named start file
|
||||
});
|
||||
|
||||
test('checkK5K6 — folder-only references fail K5 and K6', () => {
|
||||
const body = 'For detailed guidance, see references/foo/ (28 files).';
|
||||
const { K5, K6 } = checkK5K6(body, 10);
|
||||
assert.equal(K5.namedFileLinks, 0);
|
||||
assert.equal(K5.pass, false);
|
||||
assert.equal(K6.pass, false);
|
||||
});
|
||||
|
||||
test('checkRefConsistency — cited count differing from actual is a mismatch', () => {
|
||||
const body = '| `references/foo/` | 20 | desc |';
|
||||
const r = checkRefConsistency(body, { foo: 24 });
|
||||
assert.equal(r.consistent, false);
|
||||
assert.ok(r.mismatches.some((m) => m.folder === 'foo' && m.cited === 20 && m.actual === 24));
|
||||
});
|
||||
|
||||
test('checkRefConsistency — matching count is consistent', () => {
|
||||
const body = '| `references/foo/` | 24 | desc |';
|
||||
const r = checkRefConsistency(body, { foo: 24 });
|
||||
assert.equal(r.consistent, true);
|
||||
assert.deepEqual(r.mismatches, []);
|
||||
});
|
||||
|
||||
test('checkRefConsistency — same folder cited with two different numbers flags inconsistency', () => {
|
||||
const body = 'prose says references/foo/ (24 files) but table `references/foo/` | 20 |';
|
||||
const r = checkRefConsistency(body, { foo: 24 });
|
||||
assert.equal(r.consistent, false);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue