feat(ms-ai-architect): Spor D N1–N5 — Anthropic-deterministiske skill-sjekker (TDD) [skip-docs]
Legger de fem deterministiske sjekkene fra Anthropics «Skill authoring best practices» som K1–K10-rubrikken manglet, i begge lag av score-motoren: - eval.mjs (disk-laget): extractName + checkN1..checkN5 - N1 name-validitet (≤64, a-z0-9-, ingen «claude»/«anthropic»; gerund rapporteres, gater ikke) - N2 description ikke-tom + ≤1024 tegn - N3 refs én nivå dypt (ref-fil lenker ikke til annen ref-fil) - N4 TOC i ref-filer >100 linjer (delpoeng = andel store filer med TOC) - N5 forward-slash-stier (ingen Windows-backslash-stier i body) wiret inn i evalSkill.deterministic (leser nå ref-fil-innhold for N3/N4) - lib/skill-score.mjs (ren): N1–N5 i CRITERIA (vekt 1, ikke-gulv, det) Live-sanity mot de 5 skills: N1/N2/N5 rene, N3 fanger ekte nøstet ref i advisor (licensing-matrix.md → 3 andre ref-filer), N4 avdekker 387 store ref-filer uten TOC (primær forbedringsspak for oppgraderingsfasen). Tester: kb-eval 110→134 (+24). validate 239, kb-update 316 uendret grønt. [skip-docs]: N1–N5 alt spesifisert i docs/skill-quality-scoring-plan.md §2. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
bab9f2d23d
commit
e87289d929
4 changed files with 323 additions and 0 deletions
|
|
@ -68,6 +68,15 @@ export function splitFrontmatter(content) {
|
|||
};
|
||||
}
|
||||
|
||||
/** Extract the `name:` scalar from frontmatter (single-line value). */
|
||||
export function extractName(frontmatter) {
|
||||
for (const line of frontmatter.split('\n')) {
|
||||
const m = /^name:\s*(.+?)\s*$/.exec(line);
|
||||
if (m) return m[1].replace(/^["']|["']$/g, '').trim();
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
/** Extract the (possibly folded) description value from frontmatter. */
|
||||
export function extractDescription(frontmatter) {
|
||||
const lines = frontmatter.split('\n');
|
||||
|
|
@ -162,15 +171,107 @@ function scanK9Hints(body) {
|
|||
return { timeSensitiveTokenHits: tokens.length, sample: [...new Set(tokens)].slice(0, 12) };
|
||||
}
|
||||
|
||||
// --- N1-N5 — deterministic checks from Anthropic "Skill authoring best
|
||||
// practices" that the K1-K10 rubric did not yet cover. See
|
||||
// docs/skill-quality-scoring-plan.md §1 for exact rules + sources.
|
||||
|
||||
const N1_MAX_NAME_LEN = 64; // best-practices §Skill structure
|
||||
const N1_RESERVED = ['claude', 'anthropic'];
|
||||
const N2_MAX_DESC_LEN = 1024; // best-practices §Writing effective descriptions
|
||||
const N4_LARGE_FILE_LINES = 100; // best-practices §Structure longer reference files
|
||||
|
||||
/** N1 — name validity: ≤64 chars, [a-z0-9-], no reserved words. Gerund is
|
||||
* Anthropic-recommended but NOT required, so it is reported, never gated. */
|
||||
export function checkN1(name) {
|
||||
const n = (name || '').trim();
|
||||
const withinLength = n.length > 0 && n.length <= N1_MAX_NAME_LEN;
|
||||
const validChars = /^[a-z0-9-]+$/.test(n);
|
||||
const lower = n.toLowerCase();
|
||||
const noReserved = !N1_RESERVED.some((w) => lower.includes(w));
|
||||
const gerund = /[a-z]ing(?:-|$)/.test(lower); // informational only
|
||||
return { name: n, length: n.length, withinLength, validChars, noReserved, gerund, pass: withinLength && validChars && noReserved };
|
||||
}
|
||||
|
||||
/** N2 — description present and within the 1024-char hard limit. */
|
||||
export function checkN2(description) {
|
||||
const d = description || '';
|
||||
const nonEmpty = d.trim().length > 0;
|
||||
const withinLimit = d.length <= N2_MAX_DESC_LEN;
|
||||
return { length: d.length, nonEmpty, withinLimit, pass: nonEmpty && withinLimit };
|
||||
}
|
||||
|
||||
/** Markdown links to a local .md file (nesting candidates). Skips external
|
||||
* URLs and pure in-page anchors (#section). */
|
||||
function localMdLinks(content) {
|
||||
const out = [];
|
||||
const re = /\]\(([^)\s]+?\.md)(?:#[^)]*)?\)/g;
|
||||
let m;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
const target = m[1];
|
||||
if (/^[a-z][a-z0-9+.-]*:\/\//i.test(target)) continue; // external URL
|
||||
out.push(target);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** N3 — references stay one level deep: a reference file must not link to
|
||||
* another reference (.md) file. Input: [{ path, content }]. */
|
||||
export function checkN3(refFiles) {
|
||||
const nested = [];
|
||||
for (const f of refFiles) {
|
||||
const links = localMdLinks(f.content);
|
||||
if (links.length > 0) nested.push({ file: f.path, links: [...new Set(links)].slice(0, 8) });
|
||||
}
|
||||
return { checked: refFiles.length, nested, pass: nested.length === 0 };
|
||||
}
|
||||
|
||||
/** Does a reference file carry a table of contents? Either an explicit TOC
|
||||
* heading or a cluster (≥3) of in-page anchor links near the top. */
|
||||
function hasToc(content) {
|
||||
if (/^#{1,3}\s*(table of contents|contents|innhold(?:sfortegnelse)?|toc)\b/im.test(content)) return true;
|
||||
const head = content.split('\n').slice(0, 40).join('\n');
|
||||
return (head.match(/\]\(#/g) || []).length >= 3;
|
||||
}
|
||||
|
||||
/** N4 — reference files longer than 100 lines should have a table of contents.
|
||||
* Sub-score = fraction of large files that have one (vacuous pass if none). */
|
||||
export function checkN4(refFiles) {
|
||||
const large = refFiles.filter((f) => f.content.replace(/\n+$/, '').split('\n').length > N4_LARGE_FILE_LINES);
|
||||
const withToc = large.filter((f) => hasToc(f.content));
|
||||
const ratio = large.length > 0 ? Number((withToc.length / large.length).toFixed(4)) : 1;
|
||||
return {
|
||||
largeFiles: large.length,
|
||||
withToc: withToc.length,
|
||||
ratio,
|
||||
missing: large.filter((f) => !hasToc(f.content)).map((f) => f.path).slice(0, 12),
|
||||
pass: large.length === 0 || withToc.length === large.length,
|
||||
};
|
||||
}
|
||||
|
||||
/** N5 — forward-slash paths only (no Windows-style backslash paths) in body. */
|
||||
export function checkN5(body) {
|
||||
const patterns = [
|
||||
/[A-Za-z]:\\/g, // drive-letter path: C:\
|
||||
/[\w-]+\\[\w-]+\.md/g, // backslash path ending in a .md file
|
||||
/\\(references|skills|scripts|docs|assets)\b/gi, // backslash before a known dir
|
||||
];
|
||||
const hits = new Set();
|
||||
for (const re of patterns) for (const m of body.match(re) || []) hits.add(m);
|
||||
const windowsPaths = [...hits].slice(0, 12);
|
||||
return { windowsPaths, pass: windowsPaths.length === 0 };
|
||||
}
|
||||
|
||||
export function evalSkill(skillName) {
|
||||
const skillDir = join(SKILLS_DIR, skillName);
|
||||
const skillMd = join(skillDir, 'SKILL.md');
|
||||
if (!existsSync(skillMd)) return null;
|
||||
const content = readFileSync(skillMd, 'utf8');
|
||||
const { frontmatter, body } = splitFrontmatter(content);
|
||||
const name = extractName(frontmatter);
|
||||
const description = extractDescription(frontmatter);
|
||||
const refDir = join(skillDir, 'references');
|
||||
const refFiles = listMarkdown(refDir);
|
||||
const refFileContents = refFiles.map((p) => ({ path: relative(PLUGIN_ROOT, p), content: readFileSync(p, 'utf8') }));
|
||||
const perFolder = refCountsPerFolder(refDir);
|
||||
|
||||
const K2 = checkK2(description);
|
||||
|
|
@ -178,6 +279,11 @@ export function evalSkill(skillName) {
|
|||
const { K5, K6 } = checkK5K6(body, refFiles.length);
|
||||
const refConsistency = checkRefConsistency(body, perFolder);
|
||||
const K9hints = scanK9Hints(body);
|
||||
const N1 = checkN1(name);
|
||||
const N2 = checkN2(description);
|
||||
const N3 = checkN3(refFileContents);
|
||||
const N4 = checkN4(refFileContents);
|
||||
const N5 = checkN5(body);
|
||||
|
||||
return {
|
||||
name: skillName,
|
||||
|
|
@ -191,6 +297,11 @@ export function evalSkill(skillName) {
|
|||
K6_routingTable: K6,
|
||||
refCountConsistency: refConsistency,
|
||||
K9_timeSensitiveHints: K9hints,
|
||||
N1_nameValidity: N1,
|
||||
N2_descriptionLength: N2,
|
||||
N3_refNestingDepth: N3,
|
||||
N4_refToc: N4,
|
||||
N5_forwardSlashPaths: N5,
|
||||
},
|
||||
// Pointers for the LLM-judge pass (K1/K4/K7/K8/K9). The judge reads the files
|
||||
// directly; we only carry the description + sampled ref files here.
|
||||
|
|
|
|||
|
|
@ -126,6 +126,43 @@ export const CRITERIA = [
|
|||
return { available: true, sub: k.consistent ? 1 : 0, pass: !!k.consistent, detail: k.consistent ? 'OK' : `${(k.mismatches || []).length} avvik` };
|
||||
},
|
||||
},
|
||||
{
|
||||
key: 'N1', label: 'name-validitet', weight: 1, floor: false, source: 'det',
|
||||
fix: 'Rett skill-name: ≤64 tegn, kun a-z0-9-, ingen «claude»/«anthropic».',
|
||||
read: (e) => readBinary(e.deterministic?.N1_nameValidity),
|
||||
},
|
||||
{
|
||||
key: 'N2', label: 'description ≤1024', weight: 1, floor: false, source: 'det',
|
||||
fix: 'Hold description ikke-tom og ≤1024 tegn.',
|
||||
read: (e) => readBinary(e.deterministic?.N2_descriptionLength),
|
||||
},
|
||||
{
|
||||
key: 'N3', label: 'refs én nivå dypt', weight: 1, floor: false, source: 'det',
|
||||
fix: 'Fjern lenker fra ref-filer til andre ref-filer (unngå nøstede referanser).',
|
||||
read: (e) => {
|
||||
const k = e.deterministic?.N3_refNestingDepth;
|
||||
if (!k) return { available: false };
|
||||
return { available: true, sub: k.pass ? 1 : 0, pass: !!k.pass, detail: k.pass ? 'OK' : `${(k.nested || []).length} nøstede` };
|
||||
},
|
||||
},
|
||||
{
|
||||
key: 'N4', label: 'TOC i store ref-filer', weight: 1, floor: false, source: 'det',
|
||||
fix: 'Legg innholdsfortegnelse i ref-filer >100 linjer (eksplisitt TOC-heading eller anker-cluster).',
|
||||
read: (e) => {
|
||||
const k = e.deterministic?.N4_refToc;
|
||||
if (!k) return { available: false };
|
||||
return { available: true, sub: clamp01(num(k.ratio)), pass: !!k.pass, detail: `${num(k.withToc)}/${num(k.largeFiles)} store filer med TOC` };
|
||||
},
|
||||
},
|
||||
{
|
||||
key: 'N5', label: 'forward-slash-stier', weight: 1, floor: false, source: 'det',
|
||||
fix: 'Erstatt Windows-stier (\\) med forward-slash i SKILL.md.',
|
||||
read: (e) => {
|
||||
const k = e.deterministic?.N5_forwardSlashPaths;
|
||||
if (!k) return { available: false };
|
||||
return { available: true, sub: k.pass ? 1 : 0, pass: !!k.pass, detail: k.pass ? 'OK' : `${(k.windowsPaths || []).length} Windows-stier` };
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
function num(v) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue