ms-ai-architect/tests/kb-eval/test-skill-ops-apply.test.mjs

// tests/kb-eval/test-skill-ops-apply.test.mjs
// Spor B / B3 (Sesjon 18): the APPLY-PATH — the destructive frontier. An
// operator-approved (status:approved) ledger entry is executed into a real
// skills/ mutation. This file covers the SINGLE-SKILL ops (sanitize + retire);
// merge-apply (cross-skill, Sesjon 19) lives in test-skill-ops-merge-apply.test.mjs;
// create_skill comes last.
//
// Binding contract proven here:
//   1. revalidateApply (pure) re-checks an approved entry against a FRESH plan:
//      status must be approved, op-type must match, fresh guardrail must be ok,
//      and the fresh guardrail must deep-equal the approved snapshot (no drift).
//   2. applyApprovedAction archives BEFORE it deletes (rename is an atomic
//      archive+remove): dead content / whole skills land under archive/ first.
//   3. Disk drift since approval (or a non-approved / guardrail-failing entry)
//      ABORTS with zero mutation and the ledger left untouched.
//   4. A successful apply flips the ledger entry approved -> applied.
//
// Every scenario runs against a tmpdir fixture — NEVER the real skills/ tree.

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, rmSync, mkdirSync, writeFileSync, existsSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
  revalidateApply,
  applyApprovedAction,
  runSanitizePlan,
  runRetirePlan,
  planSanitizeSkill,
  SANITIZE_OPERATION,
  RETIRE_OPERATION,
} from '../../scripts/kb-eval/lib/skill-ops.mjs';
import {
  loadDecisions,
  saveDecisions,
  recordAction,
  actionKey,
} from '../../scripts/kb-update/lib/decisions-io.mjs';

// ---------------------------------------------------------------------------
// Fixture: a skills/ tree + agents/ dir under one root, so archive/ lands at
// the same root (rename stays on one filesystem). skill-a has a curated
// (referenced) ref + a dead (orphan) ref; skill-b is a small skill to retire.
// ---------------------------------------------------------------------------

function withFixture(fn) {
  const root = mkdtempSync(join(tmpdir(), 'skillops-apply-'));
  const skillsDir = join(root, 'skills');
  const agentsDir = join(root, 'agents');
  const dataDir = join(root, 'data');
  const mkRef = (skill, cat, file, body) => {
    const dir = join(skillsDir, skill, 'references', cat);
    mkdirSync(dir, { recursive: true });
    writeFileSync(join(dir, file), body);
  };
  const mkSkillMd = (skill, body) => {
    mkdirSync(join(skillsDir, skill), { recursive: true });
    writeFileSync(join(skillsDir, skill, 'SKILL.md'), body);
  };
  mkRef('skill-a', 'cat1', 'live.md', '# Live\n');
  mkRef('skill-a', 'cat1', 'dead.md', '# Dead\n');
  mkSkillMd('skill-a', '# skill-a\nSee references/cat1/live.md for details.\n');
  mkRef('skill-b', 'cat3', 'b1.md', '# B1\n');
  mkSkillMd('skill-b', '# skill-b\nSee references/cat3/b1.md.\n');
  mkdirSync(agentsDir, { recursive: true });
  writeFileSync(join(agentsDir, 'agent.md'), 'Reads references/cat1/live.md and references/cat3/b1.md\n');
  mkdirSync(dataDir, { recursive: true });
  try {
    return fn({ root, skillsDir, agentsDir, dataDir });
  } finally {
    rmSync(root, { recursive: true, force: true });
  }
}

// Seed an operator-approved entry by running the planner once (snapshotting its
// guardrail) and recording it with status:approved — exactly what the operator
// gate would have written after reviewing the dry-run.
function approveSanitize(skill, { skillsDir, agentsDir, dataDir }) {
  const { entry } = runSanitizePlan(skill, { skillsDir, agentsDir, dataDir, write: false, decided_at: '2026-06-20' });
  const approved = { ...entry, status: 'approved' };
  saveDecisions(recordAction(loadDecisions(dataDir), approved), dataDir);
  return approved;
}
function approveRetire(skill, { skillsDir, dataDir }) {
  const { entry } = runRetirePlan(skill, { skillsDir, dataDir, write: false, decided_at: '2026-06-20' });
  const approved = { ...entry, status: 'approved' };
  saveDecisions(recordAction(loadDecisions(dataDir), approved), dataDir);
  return approved;
}

// ===========================================================================
// (A) Pure core — revalidateApply
// ===========================================================================

// A self-consistent approved/fresh pair: same op, same guardrail snapshot, ok.
function pair(overridesApproved = {}, overridesFresh = {}) {
  const guardrail = { method: 'm', preCount: 4, deadCount: 2, removalCount: 2, ok: true };
  const approved = {
    operation_type: SANITIZE_OPERATION,
    status: 'approved',
    targets: { skill: 'skill-x' },
    guardrail,
    ...overridesApproved,
  };
  const fresh = {
    entry: { operation_type: SANITIZE_OPERATION, targets: { skill: 'skill-x' } },
    diff: {},
    guardrail: { ...guardrail },
    ...overridesFresh,
  };
  return { approved, fresh };
}

test('revalidateApply — OK: approved status, op match, fresh guardrail ok, no drift', () => {
  const { approved, fresh } = pair();
  const r = revalidateApply(approved, fresh);
  assert.equal(r.ok, true);
  assert.equal(r.drift, false);
  assert.equal(r.freshOk, true);
  assert.equal(r.reason, null);
});

test('revalidateApply — refuses an entry whose status is not approved', () => {
  const { approved, fresh } = pair({ status: 'pending' });
  const r = revalidateApply(approved, fresh);
  assert.equal(r.ok, false);
  assert.match(r.reason, /approved/i);
});

test('revalidateApply — refuses on operation_type mismatch', () => {
  const { approved, fresh } = pair({}, { entry: { operation_type: RETIRE_OPERATION, targets: {} } });
  const r = revalidateApply(approved, fresh);
  assert.equal(r.ok, false);
  assert.match(r.reason, /operation_type/i);
});

test('revalidateApply — refuses when the FRESH guardrail is not ok', () => {
  const { approved, fresh } = pair({}, { guardrail: { method: 'm', preCount: 4, deadCount: 2, removalCount: 2, ok: false } });
  const r = revalidateApply(approved, fresh);
  assert.equal(r.ok, false);
  assert.equal(r.freshOk, false);
});

test('revalidateApply — detects DRIFT: fresh guardrail differs from approved snapshot', () => {
  // Disk changed since approval: one dead file already gone -> counts differ.
  const { approved, fresh } = pair({}, { guardrail: { method: 'm', preCount: 3, deadCount: 1, removalCount: 1, ok: true } });
  const r = revalidateApply(approved, fresh);
  assert.equal(r.ok, false);
  assert.equal(r.drift, true);
  assert.match(r.reason, /drift/i);
});

// ===========================================================================
// (B) Impure executor — sanitize apply (archive-then-remove)
// ===========================================================================

test('applyApprovedAction sanitize — archives the dead ref, removes it, leaves curated untouched', () => {
  withFixture(({ root, skillsDir, agentsDir, dataDir }) => {
    const approved = approveSanitize('skill-a', { skillsDir, agentsDir, dataDir });
    const res = applyApprovedAction(approved, { pluginRoot: root, skillsDir, agentsDir, dataDir, apply: true, decided_at: '2026-06-21' });

    assert.equal(res.applied, true);
    assert.equal(res.revalidate.ok, true);

    // Dead ref archived FIRST, then gone from skills/.
    assert.ok(existsSync(join(root, 'archive/skills/skill-a/references/cat1/dead.md')), 'dead ref archived');
    assert.equal(existsSync(join(skillsDir, 'skill-a/references/cat1/dead.md')), false, 'dead ref removed from skills/');

    // Curated content untouched.
    assert.ok(existsSync(join(skillsDir, 'skill-a/references/cat1/live.md')), 'curated ref preserved');
    assert.ok(existsSync(join(skillsDir, 'skill-a/SKILL.md')), 'SKILL.md preserved');

    // Ledger flipped approved -> applied.
    assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'applied');
  });
});

// ===========================================================================
// (C) Impure executor — retire apply (whole-skill archive, then dir removal)
// ===========================================================================

test('applyApprovedAction retire — archives the whole skill, then removes its dir; siblings untouched', () => {
  withFixture(({ root, skillsDir, dataDir }) => {
    const approved = approveRetire('skill-b', { skillsDir, dataDir });
    const res = applyApprovedAction(approved, { pluginRoot: root, skillsDir, dataDir, apply: true, decided_at: '2026-06-21' });

    assert.equal(res.applied, true);

    // Entire skill archived (SKILL.md + ref) BEFORE the dir is removed.
    assert.ok(existsSync(join(root, 'archive/skills/skill-b/SKILL.md')), 'SKILL.md archived');
    assert.ok(existsSync(join(root, 'archive/skills/skill-b/references/cat3/b1.md')), 'ref archived');

    // skills/skill-b is gone entirely.
    assert.equal(existsSync(join(skillsDir, 'skill-b')), false, 'retired skill dir removed');

    // Sibling skill untouched.
    assert.ok(existsSync(join(skillsDir, 'skill-a/SKILL.md')), 'sibling skill survives');

    assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'applied');
  });
});

// ===========================================================================
// (D) Safety — drift, preview, non-approved, guardrail-fail, unsupported op
// ===========================================================================

test('applyApprovedAction — DRIFT after approval aborts with ZERO mutation and an untouched ledger', () => {
  withFixture(({ root, skillsDir, agentsDir, dataDir }) => {
    const approved = approveSanitize('skill-a', { skillsDir, agentsDir, dataDir });
    // Operator approved when there was 1 dead file. Now the disk drifts: the
    // dead file is already gone -> a fresh plan finds 0 dead -> guardrail differs.
    rmSync(join(skillsDir, 'skill-a/references/cat1/dead.md'));

    const res = applyApprovedAction(approved, { pluginRoot: root, skillsDir, agentsDir, dataDir, apply: true });

    assert.equal(res.applied, false, 'drift must abort the apply');
    assert.equal(res.revalidate.drift, true);
    assert.equal(existsSync(join(root, 'archive')), false, 'nothing archived on a drift abort');
    assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'approved', 'ledger left at approved');
  });
});

test('applyApprovedAction — preview (apply:false) mutates nothing and leaves the ledger at approved', () => {
  withFixture(({ root, skillsDir, agentsDir, dataDir }) => {
    const approved = approveSanitize('skill-a', { skillsDir, agentsDir, dataDir });
    const res = applyApprovedAction(approved, { pluginRoot: root, skillsDir, agentsDir, dataDir, apply: false });

    assert.equal(res.applied, false);
    assert.equal(res.preview, true);
    assert.equal(res.revalidate.ok, true, 'disk unchanged -> revalidation would pass');
    assert.ok(existsSync(join(skillsDir, 'skill-a/references/cat1/dead.md')), 'dead ref still present (preview only)');
    assert.equal(existsSync(join(root, 'archive')), false, 'preview archives nothing');
    assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'approved');
  });
});

test('applyApprovedAction — refuses a non-approved (pending) entry with zero mutation', () => {
  withFixture(({ root, skillsDir, agentsDir, dataDir }) => {
    const { entry } = runSanitizePlan('skill-a', { skillsDir, agentsDir, dataDir, write: false });
    // pending, not approved — operator never signed off.
    saveDecisions(recordAction(loadDecisions(dataDir), entry), dataDir);

    const res = applyApprovedAction(entry, { pluginRoot: root, skillsDir, agentsDir, dataDir, apply: true });
    assert.equal(res.applied, false);
    assert.match(res.revalidate.reason, /approved/i);
    assert.ok(existsSync(join(skillsDir, 'skill-a/references/cat1/dead.md')), 'no mutation');
    assert.equal(existsSync(join(root, 'archive')), false);
  });
});

test('applyApprovedAction — a guardrail-failing approved entry never applies', () => {
  withFixture(({ root, skillsDir, agentsDir, dataDir }) => {
    // Construct an approved entry whose guardrail is NOT ok (illegal removal of a
    // curated ref). A fresh plan with the same removals also fails the guardrail.
    const { entry } = planSanitizeSkill('skill-a', {
      refs: ['cat1/live.md', 'cat1/dead.md'],
      orphans: ['cat1/dead.md'],
      removals: ['cat1/live.md'], // curated -> illegal
    });
    assert.equal(entry.guardrail.ok, false, 'precondition: this dry-run already failed the guardrail');
    const approved = { ...entry, status: 'approved' };
    saveDecisions(recordAction(loadDecisions(dataDir), approved), dataDir);

    const res = applyApprovedAction(approved, { pluginRoot: root, skillsDir, agentsDir, dataDir, apply: true });
    assert.equal(res.applied, false, 'a failing guardrail must never reach the filesystem');
    assert.equal(existsSync(join(root, 'archive')), false);
    assert.ok(existsSync(join(skillsDir, 'skill-a/references/cat1/live.md')), 'curated ref still safe');
  });
});