feat(ms-ai-architect): Spor D Steg C — skill-quality SessionStart-surfacing (TDD) [skip-docs]

summarizeSkillQuality(cache) (pure) in detection-schedule.mjs: one-liner
"Skill-kvalitet: N skills < 90 % (navn)"; null when all skills meet target,
when the cache is missing (gitignored → absent in a fresh clone), or when the
shape is malformed. Reads the CACHED skill-score-report.json (from Steg B) —
never scores live (buildReport reads ~400 ref-files). Wired into
session-start-context.mjs after the course-signals block (same
existsSync/try-catch pattern). 7 new tests (kb-update 316 → 323).

[skip-docs]: README is the public end-user doc; skill-quality scoring is
maintainer-only tooling (the cache is gitignored, so the signal never surfaces
for end-users). The right doc homes ARE updated: CLAUDE.md hook table + a
caveat note (this commit), and docs/skill-quality-scoring-plan.md §3 komponent 4.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-06-23 18:38:35 +02:00
commit 97ebcd737a
4 changed files with 156 additions and 1 deletions

View file

@ -104,9 +104,17 @@ Se `references/architecture/recommended-mcp-servers.md` for detaljer.
| Event | Script | Formål |
|-------|--------|--------|
| SessionStart | `session-start-context.mjs` | Viser aktive utredninger, KB-ferskhet, onboarding-status, skill- + kurs-signaler (Spor C) + AI Act-frister |
| SessionStart | `session-start-context.mjs` | Viser aktive utredninger, KB-ferskhet, onboarding-status, skill- + kurs-signaler (Spor C), skill-kvalitetssignaler (Spor D — skills under 90 %, fra cachet score-rapport) + AI Act-frister |
| Stop | `stop-assessment-reminder.mjs` | Påminnelse om ucommittede vurderinger, neste steg |
> Skill-kvalitetssignalet (Spor D) leser den **cachede** rapporten
> `scripts/kb-eval/data/skill-score-report.json` og scorer aldri live
> (buildReport leser ~400 ref-filer — for dyrt i en SessionStart-hook).
> Cachen er gitignored (derivert/regenererbar via `score-skill.mjs --write`),
> så den er fraværende i en fersk klon: linja surfacer kun for en vedlikeholder
> som har produsert cachen lokalt, aldri for en sluttbruker. Alle skills ≥90 %
> ⇒ ingen linje.
> Secrets scanning consolidated to llm-security plugin.
## Reference docs (read on demand)

View file

@ -11,6 +11,7 @@ import {
shouldRunDetection,
summarizeSkillLifecycle,
summarizeCourses,
summarizeSkillQuality,
} from '../../scripts/kb-update/lib/detection-schedule.mjs';
import {
resolveOrgDir,
@ -203,6 +204,21 @@ if (existsSync(courseReportPath)) {
}
}
// Skill-quality score signals (Spor D / Steg C) — read-only one-liner; never
// spawns, never scores live. Reads the CACHED skill-score-report.json (written
// by `score-skill.mjs --write`). The cache is gitignored => absent in a fresh
// clone, so this surfaces only for a maintainer who has produced it locally;
// summarizeSkillQuality tolerates the missing/malformed case → null.
const scoreCachePath = join(pluginRoot, 'scripts', 'kb-eval', 'data', 'skill-score-report.json');
if (existsSync(scoreCachePath)) {
try {
const qualitySummary = summarizeSkillQuality(JSON.parse(readFileSync(scoreCachePath, 'utf8')));
if (qualitySummary) parts.push(qualitySummary);
} catch {
// Ignore — advisory only
}
}
if (nearestDeadline) {
parts.push(`EU AI Act: ${nearestDeadline.daysLeft} dager til ${nearestDeadline.label}. Kjør /architect:classify`);
}

View file

@ -256,3 +256,47 @@ export function summarizeCourses(report) {
if (changed > 0) parts.push(`${changed} endrede`);
return parts.length ? `Kurs-signaler: ${parts.join(' / ')} kurs i dekkede produkter` : null;
}
// Default skill-quality target (%). The score cache always carries its own
// `target` (buildScoreCache); this literal is only a defensive fallback for an
// older/partial cache. Mirrors TARGET in scripts/kb-eval/lib/skill-score.mjs —
// kept local so this Claude-free kb-update module never imports the kb-eval lib.
const SKILL_QUALITY_TARGET_DEFAULT = 90;
/**
* Build a compact one-liner summarizing skill-quality scores (Spor D / Steg C)
* for the SessionStart hook. Pure; tolerant of partial/missing caches. Mirrors
* summarizeSkillLifecycle / summarizeCourses.
*
* Consumes the CACHED score report (scripts/kb-eval/data/skill-score-report.json,
* produced by `score-skill.mjs --write`) it NEVER scores live: buildReport
* reads ~400 ref-files, far too expensive for a SessionStart hook. That cache is
* gitignored (derived/regenerable), so it is ABSENT in a fresh clone this MUST
* tolerate a missing/malformed cache null and never crash the hook. In
* practice the one-liner therefore surfaces only for a maintainer who has run
* `--write` locally, never for an end-user of a fresh checkout.
*
* Surfaces ONLY skills below target. Returns null when every skill meets target,
* when the cache is missing/non-object, or when it carries no usable skill data.
* @param {object|null} cache parsed skill-score-report.json
* ({target, scored:[{name, meetsTarget}], below:[name]})
* @returns {string|null}
*/
export function summarizeSkillQuality(cache) {
if (!cache || typeof cache !== 'object') return null;
const target = Number.isFinite(cache.target) ? cache.target : SKILL_QUALITY_TARGET_DEFAULT;
// Trust the precomputed `below` name-list; fall back to deriving it from
// `scored` (tolerant of an older/partial shape). Neither present => unusable.
let below;
if (Array.isArray(cache.below)) {
below = cache.below;
} else if (Array.isArray(cache.scored)) {
below = cache.scored.filter((s) => s && s.meetsTarget === false).map((s) => s.name);
} else {
return null;
}
below = below.filter(Boolean);
if (below.length === 0) return null;
const noun = below.length === 1 ? 'skill' : 'skills';
return `Skill-kvalitet: ${below.length} ${noun} < ${target} % (${below.join(', ')})`;
}

View file

@ -26,6 +26,7 @@ import {
selectDetectionSteps,
summarizeSkillLifecycle,
summarizeCourses,
summarizeSkillQuality,
daysSinceLastPoll,
} from '../../scripts/kb-update/lib/detection-schedule.mjs';
import { writeScheduleConfig } from '../../scripts/kb-update/write-schedule-config.mjs';
@ -469,6 +470,92 @@ test('summarizeCourses — tolerates a missing/partial report shape', () => {
assert.equal(summarizeCourses({ status: 'ok' }), null); // no counts
});
// ===========================================================================
// (C3) summarizeSkillQuality — skill-quality one-liner at session start (Spor D / Steg C)
// ===========================================================================
// Reads the CACHED skill-score-report.json (produced by `score-skill.mjs
// --write`), it NEVER scores live — buildReport reads ~400 ref-files, far too
// expensive for a SessionStart hook. The cache is gitignored => ABSENT in a
// fresh clone, so this MUST tolerate a missing/malformed cache → null and never
// crash the hook. Mirrors summarizeCourses / summarizeSkillLifecycle: a pure
// one-liner, null when there is nothing worth surfacing (all skills ≥ target).
const SCORE_CACHE_ALL_PASS = {
generatedAt: '2026-06-23',
target: 90,
scored: [
{ name: 'ms-ai-advisor', score: 91, meetsTarget: true },
{ name: 'ms-ai-security', score: 96, meetsTarget: true },
],
below: [],
};
const SCORE_CACHE_TWO_BELOW = {
generatedAt: '2026-06-23',
target: 90,
scored: [
{ name: 'ms-ai-advisor', score: 88, meetsTarget: false },
{ name: 'ms-ai-engineering', score: 72, meetsTarget: false },
{ name: 'ms-ai-security', score: 96, meetsTarget: true },
],
below: ['ms-ai-advisor', 'ms-ai-engineering'],
};
test('summarizeSkillQuality — surfaces the count + names of skills below target', () => {
const s = summarizeSkillQuality(SCORE_CACHE_TWO_BELOW);
assert.match(s, /Skill-kvalitet:/);
assert.match(s, /2 skills < 90 %/, 'canonical one-liner form (spec §3 komponent 4)');
assert.match(s, /ms-ai-advisor/);
assert.match(s, /ms-ai-engineering/);
});
test('summarizeSkillQuality — singular noun for exactly one skill below target', () => {
const s = summarizeSkillQuality({
target: 90,
below: ['ms-ai-advisor'],
scored: [{ name: 'ms-ai-advisor', meetsTarget: false }],
});
assert.match(s, /1 skill < 90 %/);
assert.doesNotMatch(s, /1 skills/, 'no plural "skills" for a single skill');
});
test('summarizeSkillQuality — all skills meet target returns null (nothing to surface)', () => {
assert.equal(summarizeSkillQuality(SCORE_CACHE_ALL_PASS), null);
});
test('summarizeSkillQuality — missing cache (fresh clone, gitignored) returns null', () => {
assert.equal(summarizeSkillQuality(null), null);
assert.equal(summarizeSkillQuality(undefined), null);
});
test('summarizeSkillQuality — malformed cache returns null (never crashes the hook)', () => {
assert.equal(summarizeSkillQuality({}), null, 'no below[] and no scored[] => unusable');
assert.equal(summarizeSkillQuality('nonsense'), null, 'non-object');
assert.equal(summarizeSkillQuality({ target: 90 }), null, 'target only, no skill data');
});
test('summarizeSkillQuality — uses the cache target in the message (not a hardcoded 90)', () => {
const s = summarizeSkillQuality({
target: 85,
below: ['ms-ai-advisor'],
scored: [{ name: 'ms-ai-advisor', meetsTarget: false }],
});
assert.match(s, /< 85 %/);
});
test('summarizeSkillQuality — derives below from scored when below[] is absent (shape tolerance)', () => {
const s = summarizeSkillQuality({
target: 90,
scored: [
{ name: 'ms-ai-advisor', meetsTarget: false },
{ name: 'ms-ai-security', meetsTarget: true },
],
});
assert.match(s, /1 skill < 90 %/);
assert.match(s, /ms-ai-advisor/);
assert.doesNotMatch(s, /ms-ai-security/, 'only below-target skills are named');
});
// ===========================================================================
// (D) STRUCTURAL ToS GUARANTEE — detection is Claude-free
// ===========================================================================