From 97ebcd737af5ea7d1719765b2cadd35e05b0c9c3 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 18:38:35 +0200 Subject: [PATCH] =?UTF-8?q?feat(ms-ai-architect):=20Spor=20D=20Steg=20C=20?= =?UTF-8?q?=E2=80=94=20skill-quality=20SessionStart-surfacing=20(TDD)=20[s?= =?UTF-8?q?kip-docs]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit summarizeSkillQuality(cache) (pure) in detection-schedule.mjs: one-liner "Skill-kvalitet: N skills < 90 % (navn)"; null when all skills meet target, when the cache is missing (gitignored → absent in a fresh clone), or when the shape is malformed. Reads the CACHED skill-score-report.json (from Steg B) — never scores live (buildReport reads ~400 ref-files). Wired into session-start-context.mjs after the course-signals block (same existsSync/try-catch pattern). 7 new tests (kb-update 316 → 323). [skip-docs]: README is the public end-user doc; skill-quality scoring is maintainer-only tooling (the cache is gitignored, so the signal never surfaces for end-users). The right doc homes ARE updated: CLAUDE.md hook table + a caveat note (this commit), and docs/skill-quality-scoring-plan.md §3 komponent 4. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 10 ++- hooks/scripts/session-start-context.mjs | 16 ++++ scripts/kb-update/lib/detection-schedule.mjs | 44 ++++++++++ .../test-detection-schedule.test.mjs | 87 +++++++++++++++++++ 4 files changed, 156 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index f888db1..b079bef 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -104,9 +104,17 @@ Se `references/architecture/recommended-mcp-servers.md` for detaljer. | Event | Script | Formål | |-------|--------|--------| -| SessionStart | `session-start-context.mjs` | Viser aktive utredninger, KB-ferskhet, onboarding-status, skill- + kurs-signaler (Spor C) + AI Act-frister | +| SessionStart | `session-start-context.mjs` | Viser aktive utredninger, KB-ferskhet, onboarding-status, skill- + kurs-signaler (Spor C), skill-kvalitetssignaler (Spor D — skills under 90 %, fra cachet score-rapport) + AI Act-frister | | Stop | `stop-assessment-reminder.mjs` | Påminnelse om ucommittede vurderinger, neste steg | +> Skill-kvalitetssignalet (Spor D) leser den **cachede** rapporten +> `scripts/kb-eval/data/skill-score-report.json` og scorer aldri live +> (buildReport leser ~400 ref-filer — for dyrt i en SessionStart-hook). +> Cachen er gitignored (derivert/regenererbar via `score-skill.mjs --write`), +> så den er fraværende i en fersk klon: linja surfacer kun for en vedlikeholder +> som har produsert cachen lokalt, aldri for en sluttbruker. Alle skills ≥90 % +> ⇒ ingen linje. + > Secrets scanning consolidated to llm-security plugin. ## Reference docs (read on demand) diff --git a/hooks/scripts/session-start-context.mjs b/hooks/scripts/session-start-context.mjs index 63f8d9a..c67d597 100644 --- a/hooks/scripts/session-start-context.mjs +++ b/hooks/scripts/session-start-context.mjs @@ -11,6 +11,7 @@ import { shouldRunDetection, summarizeSkillLifecycle, summarizeCourses, + summarizeSkillQuality, } from '../../scripts/kb-update/lib/detection-schedule.mjs'; import { resolveOrgDir, @@ -203,6 +204,21 @@ if (existsSync(courseReportPath)) { } } +// Skill-quality score signals (Spor D / Steg C) — read-only one-liner; never +// spawns, never scores live. Reads the CACHED skill-score-report.json (written +// by `score-skill.mjs --write`). The cache is gitignored => absent in a fresh +// clone, so this surfaces only for a maintainer who has produced it locally; +// summarizeSkillQuality tolerates the missing/malformed case → null. +const scoreCachePath = join(pluginRoot, 'scripts', 'kb-eval', 'data', 'skill-score-report.json'); +if (existsSync(scoreCachePath)) { + try { + const qualitySummary = summarizeSkillQuality(JSON.parse(readFileSync(scoreCachePath, 'utf8'))); + if (qualitySummary) parts.push(qualitySummary); + } catch { + // Ignore — advisory only + } +} + if (nearestDeadline) { parts.push(`EU AI Act: ${nearestDeadline.daysLeft} dager til ${nearestDeadline.label}. Kjør /architect:classify`); } diff --git a/scripts/kb-update/lib/detection-schedule.mjs b/scripts/kb-update/lib/detection-schedule.mjs index b1876af..386d7b1 100644 --- a/scripts/kb-update/lib/detection-schedule.mjs +++ b/scripts/kb-update/lib/detection-schedule.mjs @@ -256,3 +256,47 @@ export function summarizeCourses(report) { if (changed > 0) parts.push(`${changed} endrede`); return parts.length ? `Kurs-signaler: ${parts.join(' / ')} kurs i dekkede produkter` : null; } + +// Default skill-quality target (%). The score cache always carries its own +// `target` (buildScoreCache); this literal is only a defensive fallback for an +// older/partial cache. Mirrors TARGET in scripts/kb-eval/lib/skill-score.mjs — +// kept local so this Claude-free kb-update module never imports the kb-eval lib. +const SKILL_QUALITY_TARGET_DEFAULT = 90; + +/** + * Build a compact one-liner summarizing skill-quality scores (Spor D / Steg C) + * for the SessionStart hook. Pure; tolerant of partial/missing caches. Mirrors + * summarizeSkillLifecycle / summarizeCourses. + * + * Consumes the CACHED score report (scripts/kb-eval/data/skill-score-report.json, + * produced by `score-skill.mjs --write`) — it NEVER scores live: buildReport + * reads ~400 ref-files, far too expensive for a SessionStart hook. That cache is + * gitignored (derived/regenerable), so it is ABSENT in a fresh clone — this MUST + * tolerate a missing/malformed cache → null and never crash the hook. In + * practice the one-liner therefore surfaces only for a maintainer who has run + * `--write` locally, never for an end-user of a fresh checkout. + * + * Surfaces ONLY skills below target. Returns null when every skill meets target, + * when the cache is missing/non-object, or when it carries no usable skill data. + * @param {object|null} cache parsed skill-score-report.json + * ({target, scored:[{name, meetsTarget}], below:[name]}) + * @returns {string|null} + */ +export function summarizeSkillQuality(cache) { + if (!cache || typeof cache !== 'object') return null; + const target = Number.isFinite(cache.target) ? cache.target : SKILL_QUALITY_TARGET_DEFAULT; + // Trust the precomputed `below` name-list; fall back to deriving it from + // `scored` (tolerant of an older/partial shape). Neither present => unusable. + let below; + if (Array.isArray(cache.below)) { + below = cache.below; + } else if (Array.isArray(cache.scored)) { + below = cache.scored.filter((s) => s && s.meetsTarget === false).map((s) => s.name); + } else { + return null; + } + below = below.filter(Boolean); + if (below.length === 0) return null; + const noun = below.length === 1 ? 'skill' : 'skills'; + return `Skill-kvalitet: ${below.length} ${noun} < ${target} % (${below.join(', ')})`; +} diff --git a/tests/kb-update/test-detection-schedule.test.mjs b/tests/kb-update/test-detection-schedule.test.mjs index 62385db..e8424b6 100644 --- a/tests/kb-update/test-detection-schedule.test.mjs +++ b/tests/kb-update/test-detection-schedule.test.mjs @@ -26,6 +26,7 @@ import { selectDetectionSteps, summarizeSkillLifecycle, summarizeCourses, + summarizeSkillQuality, daysSinceLastPoll, } from '../../scripts/kb-update/lib/detection-schedule.mjs'; import { writeScheduleConfig } from '../../scripts/kb-update/write-schedule-config.mjs'; @@ -469,6 +470,92 @@ test('summarizeCourses — tolerates a missing/partial report shape', () => { assert.equal(summarizeCourses({ status: 'ok' }), null); // no counts }); +// =========================================================================== +// (C3) summarizeSkillQuality — skill-quality one-liner at session start (Spor D / Steg C) +// =========================================================================== +// Reads the CACHED skill-score-report.json (produced by `score-skill.mjs +// --write`), it NEVER scores live — buildReport reads ~400 ref-files, far too +// expensive for a SessionStart hook. The cache is gitignored => ABSENT in a +// fresh clone, so this MUST tolerate a missing/malformed cache → null and never +// crash the hook. Mirrors summarizeCourses / summarizeSkillLifecycle: a pure +// one-liner, null when there is nothing worth surfacing (all skills ≥ target). + +const SCORE_CACHE_ALL_PASS = { + generatedAt: '2026-06-23', + target: 90, + scored: [ + { name: 'ms-ai-advisor', score: 91, meetsTarget: true }, + { name: 'ms-ai-security', score: 96, meetsTarget: true }, + ], + below: [], +}; + +const SCORE_CACHE_TWO_BELOW = { + generatedAt: '2026-06-23', + target: 90, + scored: [ + { name: 'ms-ai-advisor', score: 88, meetsTarget: false }, + { name: 'ms-ai-engineering', score: 72, meetsTarget: false }, + { name: 'ms-ai-security', score: 96, meetsTarget: true }, + ], + below: ['ms-ai-advisor', 'ms-ai-engineering'], +}; + +test('summarizeSkillQuality — surfaces the count + names of skills below target', () => { + const s = summarizeSkillQuality(SCORE_CACHE_TWO_BELOW); + assert.match(s, /Skill-kvalitet:/); + assert.match(s, /2 skills < 90 %/, 'canonical one-liner form (spec §3 komponent 4)'); + assert.match(s, /ms-ai-advisor/); + assert.match(s, /ms-ai-engineering/); +}); + +test('summarizeSkillQuality — singular noun for exactly one skill below target', () => { + const s = summarizeSkillQuality({ + target: 90, + below: ['ms-ai-advisor'], + scored: [{ name: 'ms-ai-advisor', meetsTarget: false }], + }); + assert.match(s, /1 skill < 90 %/); + assert.doesNotMatch(s, /1 skills/, 'no plural "skills" for a single skill'); +}); + +test('summarizeSkillQuality — all skills meet target returns null (nothing to surface)', () => { + assert.equal(summarizeSkillQuality(SCORE_CACHE_ALL_PASS), null); +}); + +test('summarizeSkillQuality — missing cache (fresh clone, gitignored) returns null', () => { + assert.equal(summarizeSkillQuality(null), null); + assert.equal(summarizeSkillQuality(undefined), null); +}); + +test('summarizeSkillQuality — malformed cache returns null (never crashes the hook)', () => { + assert.equal(summarizeSkillQuality({}), null, 'no below[] and no scored[] => unusable'); + assert.equal(summarizeSkillQuality('nonsense'), null, 'non-object'); + assert.equal(summarizeSkillQuality({ target: 90 }), null, 'target only, no skill data'); +}); + +test('summarizeSkillQuality — uses the cache target in the message (not a hardcoded 90)', () => { + const s = summarizeSkillQuality({ + target: 85, + below: ['ms-ai-advisor'], + scored: [{ name: 'ms-ai-advisor', meetsTarget: false }], + }); + assert.match(s, /< 85 %/); +}); + +test('summarizeSkillQuality — derives below from scored when below[] is absent (shape tolerance)', () => { + const s = summarizeSkillQuality({ + target: 90, + scored: [ + { name: 'ms-ai-advisor', meetsTarget: false }, + { name: 'ms-ai-security', meetsTarget: true }, + ], + }); + assert.match(s, /1 skill < 90 %/); + assert.match(s, /ms-ai-advisor/); + assert.doesNotMatch(s, /ms-ai-security/, 'only below-target skills are named'); +}); + // =========================================================================== // (D) STRUCTURAL ToS GUARANTEE — detection is Claude-free // ===========================================================================