ms-ai-architect/tests/kb-eval/test-eval.test.mjs

// tests/kb-eval/test-eval.test.mjs
// Unit tests for the deterministic scoring functions in scripts/kb-eval/eval.mjs

import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
  splitFrontmatter,
  extractDescription,
  extractName,
  checkK2,
  checkK3,
  checkK5K6,
  checkRefConsistency,
  checkN1,
  checkN2,
  checkN3,
  checkN4,
  checkN5,
  checkCT5,
} from '../../scripts/kb-eval/eval.mjs';

test('splitFrontmatter — separates frontmatter from body', () => {
  const c = '---\nname: x\ndescription: hi\n---\n# Body\nline2\n';
  const { body } = splitFrontmatter(c);
  assert.ok(body.startsWith('# Body'), `body was: ${JSON.stringify(body)}`);
  assert.ok(!body.includes('name: x'));
});

test('splitFrontmatter — no frontmatter returns full content as body', () => {
  const { frontmatter, body } = splitFrontmatter('# No frontmatter\ntext');
  assert.equal(frontmatter, '');
  assert.equal(body, '# No frontmatter\ntext');
});

test('extractDescription — extracts folded description value', () => {
  const fm = '---\nname: x\ndescription: >-\n  Deep guidance. Triggers on: "a", "b".\nother: y\n---\n';
  const desc = extractDescription(fm);
  assert.match(desc, /Deep guidance/);
  assert.match(desc, /Triggers on/);
  assert.ok(!desc.includes('other: y'));
});

test('checkK2 — >=3 quoted phrases passes', () => {
  const r = checkK2('Some guidance. "alpha" and "beta" and "gamma".');
  assert.equal(r.quotedPhrases, 3);
  assert.equal(r.pass, true);
});

test('checkK2 — use-when form passes even without quotes', () => {
  const r = checkK2('This skill should be used when designing X.');
  assert.equal(r.useWhenForm, true);
  assert.equal(r.pass, true);
});

test('checkK2 — plain description without quotes or use-when fails', () => {
  const r = checkK2('A plain description of the domain.');
  assert.equal(r.quotedPhrases, 0);
  assert.equal(r.useWhenForm, false);
  assert.equal(r.pass, false);
});

test('checkK3 — body within 500 lines passes', () => {
  const r = checkK3('line1\nline2\nline3\n');
  assert.equal(r.bodyLines, 3);
  assert.equal(r.pass, true);
});

test('checkK3 — body over 500 lines fails', () => {
  const r = checkK3('x\n'.repeat(501));
  assert.equal(r.pass, false);
  assert.ok(r.bodyLines > 500);
});

test('checkK5K6 — named file links count toward progressive disclosure', () => {
  const body = 'See references/foo/bar.md and references/foo/baz.md for detail.';
  const { K5, K6 } = checkK5K6(body, 10);
  assert.equal(K5.namedFileLinks, 2);
  assert.equal(K5.namedRatio, 0.2);
  assert.equal(K5.pass, true); // 0.2 >= 0.20
  assert.equal(K6.pass, true); // >=1 named start file
});

test('checkK5K6 — folder-only references fail K5 and K6', () => {
  const body = 'For detailed guidance, see references/foo/ (28 files).';
  const { K5, K6 } = checkK5K6(body, 10);
  assert.equal(K5.namedFileLinks, 0);
  assert.equal(K5.pass, false);
  assert.equal(K6.pass, false);
});

test('checkRefConsistency — cited count differing from actual is a mismatch', () => {
  const body = '| `references/foo/` | 20 | desc |';
  const r = checkRefConsistency(body, { foo: 24 });
  assert.equal(r.consistent, false);
  assert.ok(r.mismatches.some((m) => m.folder === 'foo' && m.cited === 20 && m.actual === 24));
});

test('checkRefConsistency — matching count is consistent', () => {
  const body = '| `references/foo/` | 24 | desc |';
  const r = checkRefConsistency(body, { foo: 24 });
  assert.equal(r.consistent, true);
  assert.deepEqual(r.mismatches, []);
});

test('checkRefConsistency — same folder cited with two different numbers flags inconsistency', () => {
  const body = 'prose says references/foo/ (24 files) but table `references/foo/` | 20 |';
  const r = checkRefConsistency(body, { foo: 24 });
  assert.equal(r.consistent, false);
});

// --- N1-N5: deterministic checks added from Anthropic best-practices ---------

test('extractName — pulls the name scalar from frontmatter', () => {
  const fm = '---\nname: ms-ai-advisor\ndescription: x\n---\n';
  assert.equal(extractName(fm), 'ms-ai-advisor');
});

test('extractName — missing name returns empty string', () => {
  assert.equal(extractName('---\ndescription: x\n---\n'), '');
});

test('checkN1 — valid lowercase-hyphen name passes', () => {
  const r = checkN1('ms-ai-advisor');
  assert.equal(r.pass, true);
  assert.equal(r.withinLength, true);
  assert.equal(r.validChars, true);
  assert.equal(r.noReserved, true);
});

test('checkN1 — name over 64 chars fails on length', () => {
  const r = checkN1('a'.repeat(65));
  assert.equal(r.withinLength, false);
  assert.equal(r.pass, false);
});

test('checkN1 — uppercase, spaces or underscores are invalid chars', () => {
  assert.equal(checkN1('Ms-AI Advisor').validChars, false);
  assert.equal(checkN1('ms_ai_advisor').pass, false);
});

test('checkN1 — reserved words claude/anthropic fail', () => {
  assert.equal(checkN1('claude-helper').noReserved, false);
  assert.equal(checkN1('claude-helper').pass, false);
  assert.equal(checkN1('anthropic-tool').pass, false);
});

test('checkN1 — gerund is reported but does not gate pass (real skills are non-gerund)', () => {
  const r = checkN1('ms-ai-advisor');
  assert.equal(r.gerund, false);
  assert.equal(r.pass, true); // non-gerund still passes
});

test('checkN2 — non-empty description within 1024 chars passes', () => {
  const r = checkN2('A normal, useful description.');
  assert.equal(r.nonEmpty, true);
  assert.equal(r.withinLimit, true);
  assert.equal(r.pass, true);
});

test('checkN2 — empty/whitespace description fails', () => {
  const r = checkN2('   ');
  assert.equal(r.nonEmpty, false);
  assert.equal(r.pass, false);
});

test('checkN2 — description over 1024 chars fails', () => {
  const r = checkN2('x'.repeat(1025));
  assert.equal(r.withinLimit, false);
  assert.equal(r.pass, false);
});

test('checkN3 — a ref file linking to another .md flags nesting', () => {
  const refs = [{ path: 'references/a.md', content: 'see [b](b.md) for more' }];
  const r = checkN3(refs);
  assert.equal(r.pass, false);
  assert.equal(r.nested.length, 1);
  assert.equal(r.nested[0].file, 'references/a.md');
});

test('checkN3 — ref files with only external (http) links pass', () => {
  const refs = [{ path: 'references/a.md', content: 'see [docs](https://x.com/y) and [t](#anchor)' }];
  assert.equal(checkN3(refs).pass, true);
});

test('checkN3 — a ref file with no links passes', () => {
  assert.equal(checkN3([{ path: 'references/a.md', content: 'plain prose only' }]).pass, true);
});

test('checkN4 — large file without TOC fails and ratio reflects coverage', () => {
  const big = '# T\n' + 'line\n'.repeat(150);
  const r = checkN4([{ path: 'references/a.md', content: big }]);
  assert.equal(r.largeFiles, 1);
  assert.equal(r.withToc, 0);
  assert.equal(r.ratio, 0);
  assert.equal(r.pass, false);
});

test('checkN4 — large file with an explicit TOC heading passes', () => {
  const big = '# T\n\n## Innhold\n- [A](#a)\n- [B](#b)\n' + 'body\n'.repeat(150);
  const r = checkN4([{ path: 'references/a.md', content: big }]);
  assert.equal(r.withToc, 1);
  assert.equal(r.ratio, 1);
  assert.equal(r.pass, true);
});

test('checkN4 — large file with an in-page anchor-link cluster near the top passes', () => {
  const big = '# T\n- [A](#a)\n- [B](#b)\n- [C](#c)\n' + 'body\n'.repeat(150);
  const r = checkN4([{ path: 'references/a.md', content: big }]);
  assert.equal(r.withToc, 1);
  assert.equal(r.pass, true);
});

test('checkN4 — files at or under 100 lines are not required to have a TOC (vacuous pass)', () => {
  const r = checkN4([{ path: 'references/a.md', content: 'short\nfile\n' }]);
  assert.equal(r.largeFiles, 0);
  assert.equal(r.ratio, 1);
  assert.equal(r.pass, true);
});

test('checkN5 — forward-slash paths pass', () => {
  assert.equal(checkN5('See references/foo/bar.md for detail.').pass, true);
});

test('checkN5 — Windows backslash path fails', () => {
  const r = checkN5('Open references\\foo\\bar.md now.');
  assert.equal(r.pass, false);
  assert.ok(r.windowsPaths.length >= 1);
});

test('checkN5 — drive-letter path fails', () => {
  assert.equal(checkN5('Save to C:\\Users\\x\\file.md').pass, false);
});

// --- CT5 — sourcedness (deterministic, whole-corpus). REPLACES the LLM-sampled
// K8 (same signal, must not live in parallel — ref-kb-direction-note §45). Scope
// is the Port 1 contract: only files declared `type: reference` are in scope;
// CT5 = fraction of those that cite a **Source:**. template/methodology/
// regulatory are exempt (excluded from the denominator). When no file declares
// `type: reference` (unmigrated legacy corpus) the denominator is 0 — CT5 is then
// dormant (the score layer drops it; it does not tank the score).

const refFile = (over) => ({ path: 'references/x.md', content: '', ...over });
const srcLine = '**Source:** https://learn.microsoft.com/azure/x\n';

test('checkCT5 — all declared reference files cite a source → ratio 1, pass', () => {
  const refs = [
    refFile({ path: 'references/a.md', content: '# A\n**Type:** reference\n' + srcLine }),
    refFile({ path: 'references/b.md', content: '# B\n**Type:** reference\n' + srcLine }),
  ];
  const r = checkCT5(refs);
  assert.equal(r.referenceFiles, 2);
  assert.equal(r.sourced, 2);
  assert.equal(r.ratio, 1);
  assert.equal(r.pass, true);
});

test('checkCT5 — a reference file missing **Source:** lowers the ratio', () => {
  const refs = [
    refFile({ path: 'references/a.md', content: '# A\n**Type:** reference\n' + srcLine }),
    refFile({ path: 'references/b.md', content: '# B\n**Type:** reference\n' }), // no source
  ];
  const r = checkCT5(refs);
  assert.equal(r.referenceFiles, 2);
  assert.equal(r.sourced, 1);
  assert.equal(r.ratio, 0.5);
  assert.equal(r.pass, false);
});

test('checkCT5 — template/methodology/regulatory are excluded from the denominator', () => {
  const refs = [
    refFile({ path: 'references/r.md', content: '# R\n**Type:** reference\n' + srcLine }),
    refFile({ path: 'references/t.md', content: '# T\n**Type:** template\n' }),
    refFile({ path: 'references/m.md', content: '# M\n**Type:** methodology\n' }),
    refFile({ path: 'references/g.md', content: '# G\n**Type:** regulatory\n' }),
  ];
  const r = checkCT5(refs);
  assert.equal(r.referenceFiles, 1, 'only the one reference file is in scope');
  assert.equal(r.sourced, 1);
  assert.equal(r.ratio, 1);
});

test('checkCT5 — no file declares type:reference (unmigrated legacy) → referenceFiles 0', () => {
  const refs = [
    refFile({ path: 'references/a.md', content: '# A\n**Last updated:** 2026-04\n' }),
    refFile({ path: 'references/b.md', content: '# B\nplain prose, no headers\n' }),
  ];
  const r = checkCT5(refs);
  assert.equal(r.referenceFiles, 0, 'denominator empty until Port 1 migration declares types');
});

test('checkCT5 — 0.80 is the pass threshold (mirrors the old K8 ratio target)', () => {
  const mk = (n, withSrc) => Array.from({ length: n }, (_, i) =>
    refFile({ path: `references/f${i}.md`, content: '**Type:** reference\n' + (i < withSrc ? srcLine : '') }));
  assert.equal(checkCT5(mk(5, 4)).pass, true, '4/5 = 0.80 passes');
  assert.equal(checkCT5(mk(5, 3)).pass, false, '3/5 = 0.60 fails');
});