ms-ai-architect/tests/kb-eval/test-transform-criterion.test.mjs

// tests/kb-eval/test-transform-criterion.test.mjs
// Sesjon 5 ACCEPTANCE CRITERION (roadmap rad 5): "Regenerer 1 fil → eval-score ≥ baseline".
//
// Two halves, both runnable, no LLM:
//  (1) The live deterministic eval for a target skill is ≥ the recorded baseline
//      — the literal criterion, a regression guard over eval-baseline.json.
//  (2) Regenerating one REAL reference file through the lag-4 pipeline (real body
//      composed via composeKbFile) yields a file that is valid, dated, source-anchored,
//      routed back to the same skill dir, and preserves the body verbatim — so
//      writing it in place is score-preserving (same path → same ref count → same
//      deterministic eval). It is in fact strictly BETTER on two axes: the pre-lag-4
//      file has no **Source:** header (the 0%-coverage failure mode) and no TOC (N4) —
//      the regenerated one carries both (composeKbFile births a TOC for large files,
//      so `valid: true` here transitively proves the large file is TOC'd: Fase 1c).

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { dirname, join, basename } from 'node:path';
import { fileURLToPath } from 'node:url';
import { evalSkill } from '../../scripts/kb-eval/eval.mjs';
import { composeKbFile, validateKbFile, resolveTargetPath } from '../../scripts/kb-update/lib/transform.mjs';
import { loadTaxonomy } from '../../scripts/kb-update/lib/taxonomy.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const PLUGIN_ROOT = join(__dirname, '..', '..');

const baseline = JSON.parse(
  readFileSync(join(PLUGIN_ROOT, 'scripts/kb-eval/data/eval-baseline.json'), 'utf8'),
);
const TARGET = 'ms-ai-infrastructure';
const base = baseline.skills.find((s) => s.name === TARGET);

// --- (1) live deterministic eval ≥ baseline -----------------------------------

test(`live deterministic eval ≥ baseline for ${TARGET}`, () => {
  const live = evalSkill(TARGET);
  const dl = live.deterministic;
  const db = base.deterministic;
  // ref count not reduced
  assert.ok(live.refFilesActual >= base.refFilesActual,
    `refFilesActual regressed: ${live.refFilesActual} < ${base.refFilesActual}`);
  // no pass→fail regression on any deterministic check (true≥true ok, false≥true fails)
  assert.ok(dl.K2_descriptionFormat.pass >= db.K2_descriptionFormat.pass, 'K2 regressed');
  assert.ok(dl.K3_bodyLength.pass >= db.K3_bodyLength.pass, 'K3 regressed');
  assert.ok(dl.K5_progressiveDisclosure.pass >= db.K5_progressiveDisclosure.pass, 'K5 regressed');
  assert.ok(dl.K6_routingTable.pass >= db.K6_routingTable.pass, 'K6 regressed');
  assert.ok(dl.refCountConsistency.consistent >= db.refCountConsistency.consistent, 'refConsistency regressed');
});

// --- (2) regenerating one real file is score-preserving (and Source-improving) --

// Split a KB file into (header, body) at the first `---` rule line.
function splitBody(content) {
  const m = content.match(/\n---\n/);
  return m ? content.slice(m.index + m[0].length) : content;
}

test('regenerate 1 real file → valid, dated, source-anchored, body preserved, same path', () => {
  const tax = loadTaxonomy();
  const refRel = base.judgeInputs.refFileSample[0]; // a real repo-relative path
  const original = readFileSync(join(PLUGIN_ROOT, refRel), 'utf8');
  const body = splitBody(original);

  // The pre-lag-4 file has no **Source:** header — the 0%-coverage failure mode.
  assert.ok(validateKbFile(original).missing.includes('source'),
    'expected the pre-lag-4 file to lack a header Source');

  // category key + filename from the real path: skills/<skill>/references/<category>/<file>
  const parts = refRel.split('/');
  const category = parts[3];
  const filename = basename(refRel);

  const regenerated = composeKbFile({
    title: 'AI Foundry Disaster Recovery Planning',
    status: 'GA',
    category,
    source: 'https://learn.microsoft.com/azure/ai-foundry/concepts/disaster-recovery',
    lastUpdated: '2026-06',
  }, body);

  // Strictly better than the original: now valid (Source present + TOC for the large
  // body), still dated. validateKbFile requires a TOC on large files, so valid:true
  // here is also the Fase 1c anti-regression proof.
  const v = validateKbFile(regenerated);
  assert.equal(v.valid, true, `regenerated file invalid: ${v.missing}`);

  // Body preserved verbatim — transformation does not drop content.
  assert.ok(regenerated.includes(body), 'body not preserved');
  // Spot-check a stable body marker. ("Azure AI Foundry" was the old product name;
  // the file now uses "Microsoft Foundry"/"AI Foundry" after the rename.)
  assert.ok(regenerated.includes('AI Foundry'), 'expected body content missing');

  // Routes back to the SAME skill/category dir → in-place → count-preserving → eval-score preserved.
  assert.equal(resolveTargetPath(tax, category, filename), refRel);
});