ms-ai-architect/tests/kb-eval/test-skill-score-gate.test.mjs

// tests/kb-eval/test-skill-score-gate.test.mjs
// Tests for the Spor D, STEG B lifecycle-gate building blocks:
//   - lib/skill-score.mjs gateSkill(report, name): the PURE post-mutation verdict
//     used by apply-skill-op.mjs to re-score the affected skill incrementally.
//     The deterministic floor (K10) is always enforced; an unjudged skill is
//     flagged `provisional` (the K1 floor cannot be enforced -> nudge to re-judge).
//   - lib/skill-score.mjs buildScoreCache(result): the compact, PURE cache shape
//     persisted as data/skill-score-report.json and consumed by STEG C surfacing.
//   - the score-skill.mjs CLI --write flag: persists the whole-corpus cache.

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { readFileSync, rmSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
import { tmpdir } from 'node:os';
import { gateSkill, buildScoreCache, scoreReport } from '../../scripts/kb-eval/lib/skill-score.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const CLI = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'score-skill.mjs');

// A synthetic skill whose deterministic criteria all pass; K10 + judge are
// parametrizable so a single helper drives the meets/blocked/provisional cases.
function skill(name, { k10Pass = true, k10Combined = 1, worst = 'other', judge = null } = {}) {
  return {
    name,
    deterministic: {
      K2_descriptionFormat: { pass: true },
      K3_bodyLength: { bodyLines: 100, pass: true },
      K5_progressiveDisclosure: { namedRatio: 0.5, pass: true },
      K6_routingTable: { pass: true },
      refCountConsistency: { consistent: true },
      K10_siblingScopeOverlap: { maxCombined: k10Combined, pass: k10Pass, threshold: 7, worstSibling: worst },
      N1_nameValidity: { pass: true },
      N2_descriptionLength: { pass: true },
      N3_refNestingDepth: { pass: true },
      N4_refToc: { ratio: 1, pass: true, withToc: 1, largeFiles: 1 },
      N5_forwardSlashPaths: { pass: true },
    },
    judge,
  };
}

const FULL_JUDGE = {
  K1_triggerPrecision: { precision: 1, pass: true },
  K4_noDuplication: { score: 5, pass: true },
  K9_noTimeSensitive: { pass: true, findings: [] },
  K7_imperativeStyle: { ratio: 1, pass: true },
};

test('gateSkill — affected skill that meets the target is not blocked', () => {
  const report = { skills: [skill('alpha', { judge: FULL_JUDGE }), skill('beta', { judge: FULL_JUDGE })] };
  const v = gateSkill(report, 'alpha', { target: 90 });
  assert.equal(v.found, true);
  assert.equal(v.meetsTarget, true);
  assert.equal(v.blocked, false);
  assert.equal(v.provisional, false);
});

test('gateSkill — a failing K10 floor blocks the affected skill (deterministic, enforced immediately)', () => {
  const report = {
    skills: [skill('good', { judge: FULL_JUDGE }), skill('bad', { k10Pass: false, k10Combined: 9, worst: 'good', judge: FULL_JUDGE })],
  };
  const v = gateSkill(report, 'bad', { target: 90 });
  assert.equal(v.found, true);
  assert.equal(v.meetsTarget, false);
  assert.equal(v.blocked, true);
  assert.ok(v.score <= 89, `K10 floor must cap below 90, got ${v.score}`);
  assert.ok(v.improvements.some((i) => i.key === 'K10'), 'K10 must surface as an improvement');
});

test('gateSkill — an unjudged skill is provisional but the deterministic floor still blocks it', () => {
  // judge:null -> K1 floor cannot be enforced (provisional), but K10 (deterministic) still blocks.
  const report = { skills: [skill('good'), skill('bad', { k10Pass: false, k10Combined: 9, worst: 'good' })] };
  const blocked = gateSkill(report, 'bad', { target: 90 });
  assert.equal(blocked.provisional, true, 'missing judge -> provisional');
  assert.equal(blocked.blocked, true, 'deterministic K10 floor still blocks when provisional');

  const ok = gateSkill(report, 'good', { target: 90 });
  assert.equal(ok.provisional, true, 'missing judge -> provisional even when passing');
  assert.equal(ok.meetsTarget, true, 'deterministic-only score still passes');
});

test('gateSkill — an unknown affected skill returns found:false (no crash)', () => {
  const report = { skills: [skill('alpha', { judge: FULL_JUDGE })] };
  const v = gateSkill(report, 'does-not-exist', { target: 90 });
  assert.equal(v.found, false);
  assert.equal(v.name, 'does-not-exist');
});

test('buildScoreCache — compact whole-corpus shape with below[] as names', () => {
  const report = {
    skills: [skill('good', { judge: FULL_JUDGE }), skill('bad', { k10Pass: false, k10Combined: 9, worst: 'good', judge: FULL_JUDGE })],
  };
  const result = scoreReport(report, { target: 90 });
  const cache = buildScoreCache(result, { generatedAt: '2026-06-23' });
  assert.equal(cache.target, 90);
  assert.equal(cache.generatedAt, '2026-06-23');
  assert.equal(cache.scored.length, 2);
  // compact per-skill entry: the fields STEG C surfacing needs, not the verbose criteria[]
  const good = cache.scored.find((s) => s.name === 'good');
  assert.equal(typeof good.score, 'number');
  assert.equal(typeof good.meetsTarget, 'boolean');
  assert.ok(!('criteria' in good), 'cache must stay compact (no criteria[])');
  // below[] is a list of NAMES (cheap for the SessionStart one-liner)
  assert.deepEqual(cache.below, ['bad']);
});

test('buildScoreCache — deterministic given an injected generatedAt (no Date.now)', () => {
  const report = { skills: [skill('alpha', { judge: FULL_JUDGE })] };
  const result = scoreReport(report, { target: 90 });
  const a = buildScoreCache(result, { generatedAt: '2026-01-01' });
  const b = buildScoreCache(result, { generatedAt: '2026-01-01' });
  assert.deepEqual(a, b);
});

test('CLI --write <path> — persists the whole-corpus cache (all 5 skills) to disk', () => {
  const out = join(tmpdir(), `skill-score-cache-${process.pid}.json`);
  try {
    execFileSync('node', [CLI, '--write', out], { encoding: 'utf8' });
    const cache = JSON.parse(readFileSync(out, 'utf8'));
    assert.equal(cache.target, 90);
    assert.equal(cache.scored.length, 5, 'cache must cover the whole corpus, not a filtered view');
    assert.ok(Array.isArray(cache.below), 'below[] present');
    for (const s of cache.scored) {
      assert.equal(typeof s.name, 'string');
      assert.equal(typeof s.score, 'number');
      assert.equal(typeof s.meetsTarget, 'boolean');
    }
  } finally {
    rmSync(out, { force: true });
  }
});

test('CLI --write with --skill still writes the whole corpus (cache is corpus-wide)', () => {
  const out = join(tmpdir(), `skill-score-cache-corpus-${process.pid}.json`);
  try {
    execFileSync('node', [CLI, '--skill', 'ms-ai-advisor', '--write', out], { encoding: 'utf8' });
    const cache = JSON.parse(readFileSync(out, 'utf8'));
    assert.equal(cache.scored.length, 5, '--skill scopes stdout, not the persisted cache');
  } finally {
    rmSync(out, { force: true });
  }
});