ms-ai-architect/tests/kb-eval/test-score-skill.test.mjs

// tests/kb-eval/test-score-skill.test.mjs
// Tests for the score-skill CLI building blocks (Spor D, step 2):
//   - eval.mjs buildReport() assembles the full corpus report (deterministic +
//     K10 sibling-overlap + merged judge cache).
//   - lib/skill-score.mjs formatScoreReport() renders the sorted improvement
//     report (pure).
//   - the score-skill.mjs CLI: --json emits the scoreReport, --gate sets a
//     non-zero exit when any skill is below target, --skill scopes the output.

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
import { buildReport } from '../../scripts/kb-eval/eval.mjs';
import { scoreReport, formatScoreReport } from '../../scripts/kb-eval/lib/skill-score.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const CLI = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'score-skill.mjs');

test('buildReport — assembles the full corpus with N1-N5 on every skill', () => {
  const report = buildReport();
  assert.ok(Array.isArray(report.skills) && report.skills.length === 5, `got ${report.skills?.length} skills`);
  for (const s of report.skills) {
    const d = s.deterministic;
    assert.ok(d.N1_nameValidity && d.N4_refToc, `${s.name} missing N1/N4`);
    assert.ok(d.K10_siblingScopeOverlap, `${s.name} missing K10 (sibling overlap not attached)`);
  }
});

test('scoreReport — over the real corpus returns a score per skill and a below-target set', () => {
  const result = scoreReport(buildReport());
  assert.equal(result.target, 90);
  assert.equal(result.scored.length, 5);
  for (const s of result.scored) {
    assert.ok(Number.isFinite(s.score), `${s.name} score not finite`);
    assert.equal(typeof s.meetsTarget, 'boolean');
  }
  // below = subset that does not meet the target
  assert.ok(result.below.every((s) => !s.meetsTarget));
});

test('scoreReport — a synthetic below-target skill lands in below[]', () => {
  const synthetic = {
    skills: [
      // one perfect-ish, one with a failing K10 floor (capped below 90)
      {
        name: 'good', deterministic: {
          K2_descriptionFormat: { pass: true }, K3_bodyLength: { bodyLines: 100, pass: true },
          K5_progressiveDisclosure: { namedRatio: 0.5, pass: true }, K6_routingTable: { pass: true },
          refCountConsistency: { consistent: true }, K10_siblingScopeOverlap: { maxCombined: 1, pass: true, threshold: 7 },
          N1_nameValidity: { pass: true }, N2_descriptionLength: { pass: true },
          N3_refNestingDepth: { pass: true }, N4_refToc: { ratio: 1, pass: true, withToc: 1, largeFiles: 1 },
          N5_forwardSlashPaths: { pass: true },
        }, judge: null,
      },
      {
        name: 'bad', deterministic: {
          K2_descriptionFormat: { pass: true }, K3_bodyLength: { bodyLines: 100, pass: true },
          K5_progressiveDisclosure: { namedRatio: 0.5, pass: true }, K6_routingTable: { pass: true },
          refCountConsistency: { consistent: true }, K10_siblingScopeOverlap: { maxCombined: 9, pass: false, threshold: 7, worstSibling: 'good' },
          N1_nameValidity: { pass: true }, N2_descriptionLength: { pass: true },
          N3_refNestingDepth: { pass: true }, N4_refToc: { ratio: 1, pass: true, withToc: 1, largeFiles: 1 },
          N5_forwardSlashPaths: { pass: true },
        }, judge: null,
      },
    ],
  };
  const result = scoreReport(synthetic);
  assert.equal(result.below.length, 1);
  assert.equal(result.below[0].name, 'bad');
});

test('formatScoreReport — renders skill names, the target, and improvement fixes', () => {
  const result = scoreReport(buildReport());
  const text = formatScoreReport(result);
  assert.match(text, /ms-ai-advisor/);
  assert.match(text, /90/); // the target appears
  // at least one improvement fix line is rendered for a below-target skill
  if (result.below.length > 0) {
    assert.ok(/↳|→/.test(text), 'expected improvement bullets in the report');
  }
});

test('CLI --json — emits a parseable scoreReport with five scored skills (exit 0)', () => {
  const out = execFileSync('node', [CLI, '--json'], { encoding: 'utf8' });
  const parsed = JSON.parse(out);
  assert.equal(parsed.scored.length, 5);
  assert.equal(parsed.target, 90);
});

test('CLI --skill — scopes the output to a single skill', () => {
  const out = execFileSync('node', [CLI, '--skill', 'ms-ai-advisor', '--json'], { encoding: 'utf8' });
  const parsed = JSON.parse(out);
  assert.equal(parsed.scored.length, 1);
  assert.equal(parsed.scored[0].name, 'ms-ai-advisor');
});

test('CLI --gate — non-zero exit path: an unreachable target fails the gate (corpus-independent)', () => {
  let code = 0;
  try {
    execFileSync('node', [CLI, '--gate', '101'], { encoding: 'utf8', stdio: 'pipe' });
  } catch (e) {
    code = e.status;
  }
  assert.equal(code, 1, 'no skill can score 101 -> gate must fail');
});

test('CLI --gate — zero exit path: a target of 0 passes the gate (corpus-independent)', () => {
  // exits 0; execFileSync throws only on non-zero, so reaching here is the pass
  execFileSync('node', [CLI, '--gate', '0'], { encoding: 'utf8', stdio: 'pipe' });
  assert.ok(true);
});