ms-ai-architect/tests/kb-eval/test-skill-score.test.mjs

// tests/kb-eval/test-skill-score.test.mjs
// Unit tests for the PURE skill-quality scoring lib (Spor D).
// scoreSkill(evalObj) consumes the object shape produced by eval.mjs's
// evalSkill() + attachSiblingOverlap() + merged judge results, and returns a
// weighted 0-100 score with a hard floor on the load-bearing criteria (K1, K10).
// See docs/skill-quality-scoring-plan.md §2 for the rubric->score mapping.

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { scoreSkill, TARGET, FLOOR_CAP, CRITERIA } from '../../scripts/kb-eval/lib/skill-score.mjs';

// --- fixture builder: a fully-passing eval object (all sub-scores = 1) -------
function perfectEval(overrides = {}) {
  const base = {
    name: 'ms-ai-test',
    deterministic: {
      K2_descriptionFormat: { pass: true },
      K3_bodyLength: { bodyLines: 120, pass: true },
      K5_progressiveDisclosure: { namedRatio: 0.5, pass: true },
      K6_routingTable: { pass: true },
      refCountConsistency: { consistent: true, mismatches: [] },
      K9_timeSensitiveHints: { timeSensitiveTokenHits: 0 },
      K10_siblingScopeOverlap: { maxCombined: 2.0, worstSibling: 'x', pass: true, threshold: 7.0 },
      N1_nameValidity: { pass: true },
      N2_descriptionLength: { pass: true, length: 500 },
      N3_refNestingDepth: { pass: true, nested: [] },
      N4_refToc: { largeFiles: 5, withToc: 5, ratio: 1, pass: true },
      N5_forwardSlashPaths: { pass: true, windowsPaths: [] },
      CT5_sourcedness: { referenceFiles: 5, sourced: 5, ratio: 1, pass: true },
    },
    judgeInputs: { description: 'desc', bodyLines: 120 },
    judge: {
      K1_triggerPrecision: { precision: 1, pass: true },
      K4_noDuplication: { score: 5, pass: true },
      K7_imperativeStyle: { ratio: 1, pass: true },
      K9_noTimeSensitive: { pass: true, findings: [] },
    },
  };
  // shallow-merge deterministic + judge so a test can override one criterion
  return {
    ...base,
    ...overrides,
    deterministic: { ...base.deterministic, ...(overrides.deterministic || {}) },
    judge: overrides.judge === null ? null : { ...base.judge, ...(overrides.judge || {}) },
  };
}

test('scoreSkill — a fully-passing skill scores 100 and meets the target', () => {
  const r = scoreSkill(perfectEval());
  assert.equal(r.score, 100);
  assert.equal(r.meetsTarget, true);
  assert.equal(r.judged, true);
  assert.equal(r.floored, false);
  assert.equal(r.improvements.length, 0);
});

test('scoreSkill — TARGET is 90 and FLOOR_CAP is below it', () => {
  assert.equal(TARGET, 90);
  assert.ok(FLOOR_CAP < TARGET);
});

// CT5 (sourcedness) replaced the LLM-sampled K8. On an unmigrated corpus
// (referenceFiles: 0) it is dormant — dropped from the weighted mean, so it
// never tanks the score before Port 1 migration. Once reference files exist it
// contributes deterministically.

test('scoreSkill — CT5 dormant (no declared reference files) is dropped, score stays 100', () => {
  const r = scoreSkill(perfectEval({
    deterministic: { CT5_sourcedness: { referenceFiles: 0, sourced: 0, ratio: 0, pass: true } },
  }));
  assert.equal(r.score, 100, 'CT5 available:false => excluded from the mean, no score impact');
  const ct5 = r.criteria.find((c) => c.key === 'CT5');
  assert.equal(ct5.available, false);
});

test('scoreSkill — CT5 contributes deterministically once reference files exist', () => {
  const r = scoreSkill(perfectEval({
    deterministic: { CT5_sourcedness: { referenceFiles: 4, sourced: 2, ratio: 0.5, pass: false } },
  }));
  assert.ok(r.score < 100, `imperfect sourcedness should cost points, got ${r.score}`);
  const ct5 = r.criteria.find((c) => c.key === 'CT5');
  assert.equal(ct5.available, true);
  assert.equal(ct5.sub, 0.5);
  assert.ok(r.improvements.some((i) => i.key === 'CT5'), 'CT5 surfaces as an improvement');
});

test('scoreSkill — CT5 is deterministic (source det), not a judge criterion', () => {
  const ct5 = CRITERIA.find((c) => c.key === 'CT5');
  assert.ok(ct5, 'CT5 exists in the rubric');
  assert.equal(ct5.source, 'det');
  assert.equal(CRITERIA.find((c) => c.key === 'K8'), undefined, 'K8 no longer in the rubric');
});

test('scoreSkill — failing K10 (deterministic floor) caps the score below target despite a perfect rest', () => {
  const r = scoreSkill(perfectEval({
    deterministic: { K10_siblingScopeOverlap: { maxCombined: 7.4, worstSibling: 'y', pass: false, threshold: 7.0 } },
  }));
  assert.equal(r.score, FLOOR_CAP, 'capped at the floor');
  assert.equal(r.floored, true);
  assert.equal(r.meetsTarget, false);
  assert.ok(r.improvements.some((i) => i.key === 'K10' && i.floor === true));
});

test('scoreSkill — failing K1 (judge floor) caps the score below target', () => {
  const r = scoreSkill(perfectEval({
    judge: { K1_triggerPrecision: { precision: 0.6, pass: false } },
  }));
  assert.equal(r.score, FLOOR_CAP);
  assert.equal(r.meetsTarget, false);
  assert.ok(r.improvements.some((i) => i.key === 'K1' && i.floor === true));
});

test('scoreSkill — partial credit: K3 over-length and K4 mid-score reduce the score proportionally', () => {
  const r = scoreSkill(perfectEval({
    deterministic: { K3_bodyLength: { bodyLines: 600, pass: false } }, // sub = 1-(600-500)/500 = 0.8
    judge: { K4_noDuplication: { score: 3, pass: false } },            // sub = 3/5 = 0.6
  }));
  // Neither K3 nor K4 is a floor criterion, so the score is the weighted mean, not capped.
  assert.ok(r.score < 100 && r.score > FLOOR_CAP, `expected high-but-imperfect, got ${r.score}`);
  assert.equal(r.floored, false);
  const k3 = r.criteria.find((c) => c.key === 'K3');
  const k4 = r.criteria.find((c) => c.key === 'K4');
  assert.ok(Math.abs(k3.sub - 0.8) < 1e-9, `K3 sub was ${k3.sub}`);
  assert.ok(Math.abs(k4.sub - 0.6) < 1e-9, `K4 sub was ${k4.sub}`);
});

test('scoreSkill — degrades gracefully when unjudged: judge criteria excluded, K10 floor still enforced', () => {
  const r = scoreSkill(perfectEval({ judge: null }));
  assert.equal(r.judged, false);
  assert.equal(r.provisional, true, 'K1 floor cannot be enforced without judge → provisional');
  // all deterministic criteria pass → score 100 over the available (det) set
  assert.equal(r.score, 100);
  // no judge criteria should appear as available
  assert.ok(!r.criteria.some((c) => c.available && c.source === 'judge'));
});

test('scoreSkill — unjudged + failing K10 still floors (deterministic floor independent of judge)', () => {
  const r = scoreSkill(perfectEval({
    judge: null,
    deterministic: { K10_siblingScopeOverlap: { maxCombined: 9.0, worstSibling: 'z', pass: false, threshold: 7.0 } },
  }));
  assert.equal(r.score, FLOOR_CAP);
  assert.equal(r.meetsTarget, false);
});

test('scoreSkill — improvements are sorted by weighted points lost (descending)', () => {
  const r = scoreSkill(perfectEval({
    deterministic: { K2_descriptionFormat: { pass: false } },   // weight 1, lose 1.0
    judge: { K4_noDuplication: { score: 0, pass: false } },     // weight 2, lose 2.0 -> first
  }));
  assert.ok(r.improvements.length >= 2);
  assert.equal(r.improvements[0].key, 'K4', 'biggest weighted loss first');
  for (let i = 1; i < r.improvements.length; i++) {
    assert.ok(r.improvements[i - 1].pointsLost >= r.improvements[i].pointsLost);
  }
});

test('scoreSkill — every improvement carries a concrete fix string', () => {
  const r = scoreSkill(perfectEval({
    deterministic: { K3_bodyLength: { bodyLines: 700, pass: false } },
  }));
  const k3 = r.improvements.find((i) => i.key === 'K3');
  assert.ok(k3 && typeof k3.fix === 'string' && k3.fix.length > 0);
});

test('scoreSkill — N1-N5 deterministic criteria are all scored on a perfect eval', () => {
  const r = scoreSkill(perfectEval());
  for (const key of ['N1', 'N2', 'N3', 'N4', 'N5']) {
    const c = r.criteria.find((x) => x.key === key);
    assert.ok(c && c.available, `${key} should be an available criterion`);
    assert.equal(c.source, 'det', `${key} is deterministic`);
    assert.equal(c.floor, false, `${key} is not a floor criterion`);
  }
  assert.equal(r.score, 100, 'all N1-N5 passing keeps the perfect score');
});

test('scoreSkill — N4 (TOC) gives partial credit proportional to coverage ratio', () => {
  const r = scoreSkill(perfectEval({
    deterministic: { N4_refToc: { largeFiles: 4, withToc: 2, ratio: 0.5, pass: false } },
  }));
  const n4 = r.criteria.find((c) => c.key === 'N4');
  assert.ok(Math.abs(n4.sub - 0.5) < 1e-9, `N4 sub was ${n4.sub}`);
  assert.equal(r.floored, false, 'N4 is not a floor criterion');
  assert.ok(r.score < 100 && r.score > FLOOR_CAP, `expected high-but-imperfect, got ${r.score}`);
  assert.ok(r.improvements.some((i) => i.key === 'N4'));
});

test('scoreSkill — an invalid name (N1) is a non-floor miss that lands in improvements', () => {
  const r = scoreSkill(perfectEval({
    deterministic: { N1_nameValidity: { pass: false } },
  }));
  const n1 = r.criteria.find((c) => c.key === 'N1');
  assert.equal(n1.sub, 0);
  assert.equal(r.floored, false, 'N1 does not floor the score');
  assert.ok(r.meetsTarget, 'a single non-floor binary miss should not drop below 90');
  assert.ok(r.improvements.some((i) => i.key === 'N1' && typeof i.fix === 'string'));
});

test('scoreSkill — nested references (N3) reduce the score', () => {
  const r = scoreSkill(perfectEval({
    deterministic: { N3_refNestingDepth: { pass: false, nested: [{ file: 'references/a.md', links: ['b.md'] }] } },
  }));
  assert.ok(r.score < 100);
  assert.ok(r.improvements.some((i) => i.key === 'N3'));
});

test('scoreSkill — tolerates a null/garbage eval object without throwing', () => {
  assert.equal(scoreSkill(null), null);
  assert.equal(scoreSkill({}), null);
});