Kjell Tore Guttormsen
215772df87
Read-only eval-verktoy scripts/kb-eval/eval.mjs (determ. K2/K3/K5/K6 + ref-tall) + operator-gated LLM-judge (judge-prompt.md -> data/judge-results.json) flettet til data/eval-baseline.json. 13 enhetstester (tests/kb-eval/), 42 kb-update-tester gronne. Baseline: K5 3/5 fail, K9 4/5 fail, K4 2/5 fail, ref-tall 2/5 fail; 2 konkrete bugs flagget (security 6x5-vekting-motstrid, governance broken ref-path). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01REiKFhP4w6xGXXqWKpPCJJ
102 lines
3.7 KiB
JavaScript
102 lines
3.7 KiB
JavaScript
// tests/kb-eval/test-eval.test.mjs
|
|
// Unit tests for the deterministic scoring functions in scripts/kb-eval/eval.mjs
|
|
|
|
import { test } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import {
|
|
splitFrontmatter,
|
|
extractDescription,
|
|
checkK2,
|
|
checkK3,
|
|
checkK5K6,
|
|
checkRefConsistency,
|
|
} from '../../scripts/kb-eval/eval.mjs';
|
|
|
|
test('splitFrontmatter — separates frontmatter from body', () => {
|
|
const c = '---\nname: x\ndescription: hi\n---\n# Body\nline2\n';
|
|
const { body } = splitFrontmatter(c);
|
|
assert.ok(body.startsWith('# Body'), `body was: ${JSON.stringify(body)}`);
|
|
assert.ok(!body.includes('name: x'));
|
|
});
|
|
|
|
test('splitFrontmatter — no frontmatter returns full content as body', () => {
|
|
const { frontmatter, body } = splitFrontmatter('# No frontmatter\ntext');
|
|
assert.equal(frontmatter, '');
|
|
assert.equal(body, '# No frontmatter\ntext');
|
|
});
|
|
|
|
test('extractDescription — extracts folded description value', () => {
|
|
const fm = '---\nname: x\ndescription: >-\n Deep guidance. Triggers on: "a", "b".\nother: y\n---\n';
|
|
const desc = extractDescription(fm);
|
|
assert.match(desc, /Deep guidance/);
|
|
assert.match(desc, /Triggers on/);
|
|
assert.ok(!desc.includes('other: y'));
|
|
});
|
|
|
|
test('checkK2 — >=3 quoted phrases passes', () => {
|
|
const r = checkK2('Some guidance. "alpha" and "beta" and "gamma".');
|
|
assert.equal(r.quotedPhrases, 3);
|
|
assert.equal(r.pass, true);
|
|
});
|
|
|
|
test('checkK2 — use-when form passes even without quotes', () => {
|
|
const r = checkK2('This skill should be used when designing X.');
|
|
assert.equal(r.useWhenForm, true);
|
|
assert.equal(r.pass, true);
|
|
});
|
|
|
|
test('checkK2 — plain description without quotes or use-when fails', () => {
|
|
const r = checkK2('A plain description of the domain.');
|
|
assert.equal(r.quotedPhrases, 0);
|
|
assert.equal(r.useWhenForm, false);
|
|
assert.equal(r.pass, false);
|
|
});
|
|
|
|
test('checkK3 — body within 500 lines passes', () => {
|
|
const r = checkK3('line1\nline2\nline3\n');
|
|
assert.equal(r.bodyLines, 3);
|
|
assert.equal(r.pass, true);
|
|
});
|
|
|
|
test('checkK3 — body over 500 lines fails', () => {
|
|
const r = checkK3('x\n'.repeat(501));
|
|
assert.equal(r.pass, false);
|
|
assert.ok(r.bodyLines > 500);
|
|
});
|
|
|
|
test('checkK5K6 — named file links count toward progressive disclosure', () => {
|
|
const body = 'See references/foo/bar.md and references/foo/baz.md for detail.';
|
|
const { K5, K6 } = checkK5K6(body, 10);
|
|
assert.equal(K5.namedFileLinks, 2);
|
|
assert.equal(K5.namedRatio, 0.2);
|
|
assert.equal(K5.pass, true); // 0.2 >= 0.20
|
|
assert.equal(K6.pass, true); // >=1 named start file
|
|
});
|
|
|
|
test('checkK5K6 — folder-only references fail K5 and K6', () => {
|
|
const body = 'For detailed guidance, see references/foo/ (28 files).';
|
|
const { K5, K6 } = checkK5K6(body, 10);
|
|
assert.equal(K5.namedFileLinks, 0);
|
|
assert.equal(K5.pass, false);
|
|
assert.equal(K6.pass, false);
|
|
});
|
|
|
|
test('checkRefConsistency — cited count differing from actual is a mismatch', () => {
|
|
const body = '| `references/foo/` | 20 | desc |';
|
|
const r = checkRefConsistency(body, { foo: 24 });
|
|
assert.equal(r.consistent, false);
|
|
assert.ok(r.mismatches.some((m) => m.folder === 'foo' && m.cited === 20 && m.actual === 24));
|
|
});
|
|
|
|
test('checkRefConsistency — matching count is consistent', () => {
|
|
const body = '| `references/foo/` | 24 | desc |';
|
|
const r = checkRefConsistency(body, { foo: 24 });
|
|
assert.equal(r.consistent, true);
|
|
assert.deepEqual(r.mismatches, []);
|
|
});
|
|
|
|
test('checkRefConsistency — same folder cited with two different numbers flags inconsistency', () => {
|
|
const body = 'prose says references/foo/ (24 files) but table `references/foo/` | 20 |';
|
|
const r = checkRefConsistency(body, { foo: 24 });
|
|
assert.equal(r.consistent, false);
|
|
});
|