ms-ai-architect/tests/kb-update/test-classify-ref-type.test.mjs

// tests/kb-update/test-classify-ref-type.test.mjs
// Spor 1 Step 2 — the pure ref-type classifier: MS-citation is the first-pass proxy
// for `reference`, but zero-MS files and methodology/template-shaped paths that DO
// cite MS are never silently classified — they carry reviewFlag:true for the Step-4
// edge-set adjudication. Pure core, no disk, deterministic output.

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { mkdtempSync, writeFileSync, readFileSync, existsSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
import {
  classifyRefType,
  classifyCorpus,
  VALID_TYPES,
} from '../../scripts/kb-update/lib/classify-ref-type.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const CLI = join(__dirname, '..', '..', 'scripts', 'kb-update', 'classify-ref-type.mjs');

const MS_CITING =
  '# Azure AI Search\n\n**Category:** rag\n\n---\n\n## A\n\n' +
  'Se https://learn.microsoft.com/azure/search/hybrid-search-overview for detaljer.\n';

const ZERO_MS =
  '# Vurderingsnotat\n\n**Category:** governance\n\n---\n\n## A\n\n' +
  'Ren intern metodetekst uten eksterne kilder.\n';

const METHODOLOGY_SHAPED_MS_CITING =
  '# Vurderingsmetodikk\n\n**Category:** governance\n\n---\n\n## A\n\n' +
  'Rammeverk-steg. Se https://learn.microsoft.com/azure/well-architected/ for bakgrunn.\n';

const REGULATORY_ZERO_MS =
  '# EU AI Act Art. 6\n\n**Category:** governance\n\n---\n\n## A\n\n' +
  'Forordningsteksten: https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:32024R1689\n';

test('classifyRefType — MS-citing file → reference, mscite:true, no review flag', () => {
  const r = classifyRefType({ path: 'skills/ms-ai-engineering/references/rag/azure-ai-search.md', content: MS_CITING });
  assert.equal(r.type, 'reference');
  assert.equal(r.mscite, true);
  assert.equal(r.reviewFlag, false);
  assert.equal(r.signal, 'ms-citation');
});

test('classifyRefType — zero-MS file is NOT silently reference: reviewFlag:true + path-heuristic candidate', () => {
  const r = classifyRefType({ path: 'skills/ms-ai-governance/references/governance/vurderingsnotat.md', content: ZERO_MS });
  assert.equal(r.mscite, false);
  assert.equal(r.reviewFlag, true);
  assert.ok(VALID_TYPES.includes(r.type));
});

test('classifyRefType — methodology-shaped path that cites MS → reviewFlag:true (opposite false-positive direction)', () => {
  const r = classifyRefType({
    path: 'skills/ms-ai-governance/references/methodology/vurderingsmetodikk.md',
    content: METHODOLOGY_SHAPED_MS_CITING,
  });
  assert.equal(r.mscite, true);
  assert.equal(r.reviewFlag, true);
  assert.ok(VALID_TYPES.includes(r.type));
});

test('classifyRefType — template-shaped path that cites MS → reviewFlag:true', () => {
  const r = classifyRefType({
    path: 'skills/ms-ai-governance/references/templates/dpia-template.md',
    content: MS_CITING,
  });
  assert.equal(r.reviewFlag, true);
  assert.ok(VALID_TYPES.includes(r.type));
});

test('classifyRefType — regulatory text citing eur-lex (zero-MS) → reviewFlag:true, regulatory candidate', () => {
  const r = classifyRefType({
    path: 'skills/ms-ai-governance/references/eu-ai-act/article-6.md',
    content: REGULATORY_ZERO_MS,
  });
  assert.equal(r.mscite, false);
  assert.equal(r.reviewFlag, true);
  assert.equal(r.type, 'regulatory');
});

test('classifyRefType — every emitted type is in VALID_TYPES', () => {
  const fixtures = [
    { path: 'skills/x/references/a.md', content: MS_CITING },
    { path: 'skills/x/references/templates/b.md', content: ZERO_MS },
    { path: 'skills/x/references/methodology/c.md', content: METHODOLOGY_SHAPED_MS_CITING },
    { path: 'skills/x/references/eu/d.md', content: REGULATORY_ZERO_MS },
  ];
  for (const f of fixtures) {
    const r = classifyRefType(f);
    assert.ok(VALID_TYPES.includes(r.type), `${r.type} not in VALID_TYPES`);
  }
});

test('classifyCorpus — relpath-keyed, key-sorted, deterministic across runs', () => {
  const entries = [
    { path: 'skills/z/references/z.md', content: MS_CITING },
    { path: 'skills/a/references/a.md', content: ZERO_MS },
    { path: 'skills/m/references/m.md', content: METHODOLOGY_SHAPED_MS_CITING },
  ];
  const m1 = classifyCorpus(entries);
  const m2 = classifyCorpus([...entries].reverse());
  assert.deepEqual(m1, m2);
  assert.equal(JSON.stringify(m1), JSON.stringify(m2));
  assert.deepEqual(Object.keys(m1), [...Object.keys(m1)].sort());
  assert.equal(Object.keys(m1).length, 3);
  for (const entry of Object.values(m1)) {
    assert.ok(VALID_TYPES.includes(entry.type));
    assert.equal(typeof entry.reviewFlag, 'boolean');
    assert.equal(typeof entry.mscite, 'boolean');
  }
});

// ---------------------------------------------------------------------------
// Step 3 — the IO-shell CLI: full-corpus classification, --write persistence
// and the manifest re-entrance guard (plan v1.8: --write refuses exit 3 on an
// enriched manifest unless --merge; --merge preserves enrichment verbatim).
// ---------------------------------------------------------------------------

// A real corpus file (stable since 2026-06) used as the enriched-entry key.
const REAL_RELPATH = 'skills/ms-ai-engineering/references/mlops-genaiops/mlops-fundamentals-overview.md';
const GONE_RELPATH = 'skills/ms-ai-engineering/references/zzz-does-not-exist.md';

// Deliberately different from what fresh classification would produce, so
// verbatim preservation is distinguishable from re-classification.
const ENRICHED_ENTRY = {
  type: 'methodology',
  signal: 'test-signal',
  mscite: true,
  reviewFlag: false,
  resolvedBy: 'human-test',
  source: 'https://learn.microsoft.com/test-authority',
};

test('CLI --json — exactly 389 relpath-keyed entries, every type valid, zero unclassified', () => {
  const out = execFileSync('node', [CLI, '--json'], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 });
  const manifest = JSON.parse(out);
  const keys = Object.keys(manifest);
  assert.equal(keys.length, 389);
  assert.deepEqual(keys, [...keys].sort());
  for (const [path, entry] of Object.entries(manifest)) {
    assert.ok(path.startsWith('skills/'), path);
    assert.ok(VALID_TYPES.includes(entry.type), `${path}: ${entry.type}`);
  }
});

test('CLI — without --write no manifest file is created', () => {
  const tmp = mkdtempSync(join(tmpdir(), 'clsrt-'));
  const out = join(tmp, 'manifest.json');
  try {
    execFileSync('node', [CLI, '--out', out], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 });
    assert.equal(existsSync(out), false);
  } finally {
    rmSync(tmp, { recursive: true, force: true });
  }
});

test('CLI --write — persists a parseable, key-sorted manifest to --out', () => {
  const tmp = mkdtempSync(join(tmpdir(), 'clsrt-'));
  const out = join(tmp, 'manifest.json');
  try {
    execFileSync('node', [CLI, '--write', '--out', out], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 });
    const manifest = JSON.parse(readFileSync(out, 'utf8'));
    assert.equal(Object.keys(manifest).length, 389);
  } finally {
    rmSync(tmp, { recursive: true, force: true });
  }
});

test('CLI --write — refuses (exit 3, file untouched) when the existing manifest is enriched', () => {
  const tmp = mkdtempSync(join(tmpdir(), 'clsrt-'));
  const out = join(tmp, 'manifest.json');
  try {
    const existing = { [REAL_RELPATH]: ENRICHED_ENTRY };
    writeFileSync(out, JSON.stringify(existing, null, 2) + '\n');
    const before = readFileSync(out, 'utf8');
    let status = 0;
    try {
      execFileSync('node', [CLI, '--write', '--out', out], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 });
    } catch (err) {
      status = err.status;
    }
    assert.equal(status, 3);
    assert.equal(readFileSync(out, 'utf8'), before);
  } finally {
    rmSync(tmp, { recursive: true, force: true });
  }
});

test('CLI --write --merge — preserves enrichment verbatim, adds disk-new, drops deleted', () => {
  const tmp = mkdtempSync(join(tmpdir(), 'clsrt-'));
  const out = join(tmp, 'manifest.json');
  try {
    const existing = {
      [REAL_RELPATH]: ENRICHED_ENTRY,
      [GONE_RELPATH]: { type: 'reference', signal: 'ms-citation', mscite: true, reviewFlag: false },
    };
    writeFileSync(out, JSON.stringify(existing, null, 2) + '\n');
    execFileSync('node', [CLI, '--write', '--merge', '--out', out], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 });
    const manifest = JSON.parse(readFileSync(out, 'utf8'));
    // deleted-on-disk entry dropped; disk-new files added → the live 389
    assert.equal(manifest[GONE_RELPATH], undefined);
    assert.equal(Object.keys(manifest).length, 389);
    // enriched entry preserved verbatim — type/reviewFlag/resolvedBy/source/signal intact
    assert.deepEqual(manifest[REAL_RELPATH], ENRICHED_ENTRY);
    const keys = Object.keys(manifest);
    assert.deepEqual(keys, [...keys].sort());
  } finally {
    rmSync(tmp, { recursive: true, force: true });
  }
});

test('CLI — unknown flag → usage error exit 2', () => {
  let status = 0;
  try {
    execFileSync('node', [CLI, '--bogus'], { encoding: 'utf8' });
  } catch (err) {
    status = err.status;
  }
  assert.equal(status, 2);
});