feat(ms-ai-architect): Sesjon 18 — B3 apply-path (sanitize+retire), merge→S19 [skip-docs]
Apply-path = den destruktive frontieren: eksekver en operatør-godkjent
(status:approved) ledger-entry → faktisk skills/-mutasjon. Staget per
operatør-beslutning til single-skill ops (sanitize+retire); merge-apply
isoleres til S19 (krever taksonomi-persistering + cross-skill flytt).
- revalidateApply (ren, skill-ops.mjs): idempotent revalidering mot fersk
re-plan — nekter ved status≠approved, op-mismatch, fersk guardrail≠ok,
eller drift (isDeepStrictEqual fresh vs approved guardrail-snapshot).
- applyApprovedAction (impur): arkiver-så-slett (rename skills/…→archive/…
atomisk; retire rmdir'er tom katalog sist) + flipp ledger approved→applied.
apply:false = preview (0 mutasjon). merge_skills → throw (S19).
- setActionStatus (ren, decisions-io.mjs): ledger status-transisjon, bevarer
targets+guardrail, audit-record beholdes.
- CLI apply-skill-op.mjs {list|sanitize|retire} — default PREVIEW, --apply =
dobbel-gate utover ledger-approved.
TDD, tmpdir-fixtures: 0 ekte skills/-mutasjon. Tester: kb-eval 66→78,
kb-update 132→137; validate 239 · kb-integrity 192/192 uendret.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
f7b622b318
commit
4ac18a76c8
5 changed files with 603 additions and 2 deletions
286
tests/kb-eval/test-skill-ops-apply.test.mjs
Normal file
286
tests/kb-eval/test-skill-ops-apply.test.mjs
Normal file
|
|
@ -0,0 +1,286 @@
|
|||
// tests/kb-eval/test-skill-ops-apply.test.mjs
|
||||
// Spor B / B3 (Sesjon 18): the APPLY-PATH — the destructive frontier. An
|
||||
// operator-approved (status:approved) ledger entry is executed into a real
|
||||
// skills/ mutation. Single-skill ops only this session (sanitize + retire);
|
||||
// merge-apply is staged to S19 (it needs taxonomy persistence + cross-skill
|
||||
// moves). create_skill comes last.
|
||||
//
|
||||
// Binding contract proven here:
|
||||
// 1. revalidateApply (pure) re-checks an approved entry against a FRESH plan:
|
||||
// status must be approved, op-type must match, fresh guardrail must be ok,
|
||||
// and the fresh guardrail must deep-equal the approved snapshot (no drift).
|
||||
// 2. applyApprovedAction archives BEFORE it deletes (rename is an atomic
|
||||
// archive+remove): dead content / whole skills land under archive/ first.
|
||||
// 3. Disk drift since approval (or a non-approved / guardrail-failing entry)
|
||||
// ABORTS with zero mutation and the ledger left untouched.
|
||||
// 4. A successful apply flips the ledger entry approved -> applied.
|
||||
//
|
||||
// Every scenario runs against a tmpdir fixture — NEVER the real skills/ tree.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, rmSync, mkdirSync, writeFileSync, existsSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
revalidateApply,
|
||||
applyApprovedAction,
|
||||
runSanitizePlan,
|
||||
runRetirePlan,
|
||||
planSanitizeSkill,
|
||||
SANITIZE_OPERATION,
|
||||
RETIRE_OPERATION,
|
||||
MERGE_OPERATION,
|
||||
} from '../../scripts/kb-eval/lib/skill-ops.mjs';
|
||||
import {
|
||||
loadDecisions,
|
||||
saveDecisions,
|
||||
recordAction,
|
||||
actionKey,
|
||||
} from '../../scripts/kb-update/lib/decisions-io.mjs';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture: a skills/ tree + agents/ dir under one root, so archive/ lands at
|
||||
// the same root (rename stays on one filesystem). skill-a has a curated
|
||||
// (referenced) ref + a dead (orphan) ref; skill-b is a small skill to retire.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function withFixture(fn) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'skillops-apply-'));
|
||||
const skillsDir = join(root, 'skills');
|
||||
const agentsDir = join(root, 'agents');
|
||||
const dataDir = join(root, 'data');
|
||||
const mkRef = (skill, cat, file, body) => {
|
||||
const dir = join(skillsDir, skill, 'references', cat);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(join(dir, file), body);
|
||||
};
|
||||
const mkSkillMd = (skill, body) => {
|
||||
mkdirSync(join(skillsDir, skill), { recursive: true });
|
||||
writeFileSync(join(skillsDir, skill, 'SKILL.md'), body);
|
||||
};
|
||||
mkRef('skill-a', 'cat1', 'live.md', '# Live\n');
|
||||
mkRef('skill-a', 'cat1', 'dead.md', '# Dead\n');
|
||||
mkSkillMd('skill-a', '# skill-a\nSee references/cat1/live.md for details.\n');
|
||||
mkRef('skill-b', 'cat3', 'b1.md', '# B1\n');
|
||||
mkSkillMd('skill-b', '# skill-b\nSee references/cat3/b1.md.\n');
|
||||
mkdirSync(agentsDir, { recursive: true });
|
||||
writeFileSync(join(agentsDir, 'agent.md'), 'Reads references/cat1/live.md and references/cat3/b1.md\n');
|
||||
mkdirSync(dataDir, { recursive: true });
|
||||
try {
|
||||
return fn({ root, skillsDir, agentsDir, dataDir });
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// Seed an operator-approved entry by running the planner once (snapshotting its
|
||||
// guardrail) and recording it with status:approved — exactly what the operator
|
||||
// gate would have written after reviewing the dry-run.
|
||||
function approveSanitize(skill, { skillsDir, agentsDir, dataDir }) {
|
||||
const { entry } = runSanitizePlan(skill, { skillsDir, agentsDir, dataDir, write: false, decided_at: '2026-06-20' });
|
||||
const approved = { ...entry, status: 'approved' };
|
||||
saveDecisions(recordAction(loadDecisions(dataDir), approved), dataDir);
|
||||
return approved;
|
||||
}
|
||||
function approveRetire(skill, { skillsDir, dataDir }) {
|
||||
const { entry } = runRetirePlan(skill, { skillsDir, dataDir, write: false, decided_at: '2026-06-20' });
|
||||
const approved = { ...entry, status: 'approved' };
|
||||
saveDecisions(recordAction(loadDecisions(dataDir), approved), dataDir);
|
||||
return approved;
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// (A) Pure core — revalidateApply
|
||||
// ===========================================================================
|
||||
|
||||
// A self-consistent approved/fresh pair: same op, same guardrail snapshot, ok.
|
||||
function pair(overridesApproved = {}, overridesFresh = {}) {
|
||||
const guardrail = { method: 'm', preCount: 4, deadCount: 2, removalCount: 2, ok: true };
|
||||
const approved = {
|
||||
operation_type: SANITIZE_OPERATION,
|
||||
status: 'approved',
|
||||
targets: { skill: 'skill-x' },
|
||||
guardrail,
|
||||
...overridesApproved,
|
||||
};
|
||||
const fresh = {
|
||||
entry: { operation_type: SANITIZE_OPERATION, targets: { skill: 'skill-x' } },
|
||||
diff: {},
|
||||
guardrail: { ...guardrail },
|
||||
...overridesFresh,
|
||||
};
|
||||
return { approved, fresh };
|
||||
}
|
||||
|
||||
test('revalidateApply — OK: approved status, op match, fresh guardrail ok, no drift', () => {
|
||||
const { approved, fresh } = pair();
|
||||
const r = revalidateApply(approved, fresh);
|
||||
assert.equal(r.ok, true);
|
||||
assert.equal(r.drift, false);
|
||||
assert.equal(r.freshOk, true);
|
||||
assert.equal(r.reason, null);
|
||||
});
|
||||
|
||||
test('revalidateApply — refuses an entry whose status is not approved', () => {
|
||||
const { approved, fresh } = pair({ status: 'pending' });
|
||||
const r = revalidateApply(approved, fresh);
|
||||
assert.equal(r.ok, false);
|
||||
assert.match(r.reason, /approved/i);
|
||||
});
|
||||
|
||||
test('revalidateApply — refuses on operation_type mismatch', () => {
|
||||
const { approved, fresh } = pair({}, { entry: { operation_type: RETIRE_OPERATION, targets: {} } });
|
||||
const r = revalidateApply(approved, fresh);
|
||||
assert.equal(r.ok, false);
|
||||
assert.match(r.reason, /operation_type/i);
|
||||
});
|
||||
|
||||
test('revalidateApply — refuses when the FRESH guardrail is not ok', () => {
|
||||
const { approved, fresh } = pair({}, { guardrail: { method: 'm', preCount: 4, deadCount: 2, removalCount: 2, ok: false } });
|
||||
const r = revalidateApply(approved, fresh);
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.freshOk, false);
|
||||
});
|
||||
|
||||
test('revalidateApply — detects DRIFT: fresh guardrail differs from approved snapshot', () => {
|
||||
// Disk changed since approval: one dead file already gone -> counts differ.
|
||||
const { approved, fresh } = pair({}, { guardrail: { method: 'm', preCount: 3, deadCount: 1, removalCount: 1, ok: true } });
|
||||
const r = revalidateApply(approved, fresh);
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.drift, true);
|
||||
assert.match(r.reason, /drift/i);
|
||||
});
|
||||
|
||||
// ===========================================================================
|
||||
// (B) Impure executor — sanitize apply (archive-then-remove)
|
||||
// ===========================================================================
|
||||
|
||||
test('applyApprovedAction sanitize — archives the dead ref, removes it, leaves curated untouched', () => {
|
||||
withFixture(({ root, skillsDir, agentsDir, dataDir }) => {
|
||||
const approved = approveSanitize('skill-a', { skillsDir, agentsDir, dataDir });
|
||||
const res = applyApprovedAction(approved, { pluginRoot: root, skillsDir, agentsDir, dataDir, apply: true, decided_at: '2026-06-21' });
|
||||
|
||||
assert.equal(res.applied, true);
|
||||
assert.equal(res.revalidate.ok, true);
|
||||
|
||||
// Dead ref archived FIRST, then gone from skills/.
|
||||
assert.ok(existsSync(join(root, 'archive/skills/skill-a/references/cat1/dead.md')), 'dead ref archived');
|
||||
assert.equal(existsSync(join(skillsDir, 'skill-a/references/cat1/dead.md')), false, 'dead ref removed from skills/');
|
||||
|
||||
// Curated content untouched.
|
||||
assert.ok(existsSync(join(skillsDir, 'skill-a/references/cat1/live.md')), 'curated ref preserved');
|
||||
assert.ok(existsSync(join(skillsDir, 'skill-a/SKILL.md')), 'SKILL.md preserved');
|
||||
|
||||
// Ledger flipped approved -> applied.
|
||||
assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'applied');
|
||||
});
|
||||
});
|
||||
|
||||
// ===========================================================================
|
||||
// (C) Impure executor — retire apply (whole-skill archive, then dir removal)
|
||||
// ===========================================================================
|
||||
|
||||
test('applyApprovedAction retire — archives the whole skill, then removes its dir; siblings untouched', () => {
|
||||
withFixture(({ root, skillsDir, dataDir }) => {
|
||||
const approved = approveRetire('skill-b', { skillsDir, dataDir });
|
||||
const res = applyApprovedAction(approved, { pluginRoot: root, skillsDir, dataDir, apply: true, decided_at: '2026-06-21' });
|
||||
|
||||
assert.equal(res.applied, true);
|
||||
|
||||
// Entire skill archived (SKILL.md + ref) BEFORE the dir is removed.
|
||||
assert.ok(existsSync(join(root, 'archive/skills/skill-b/SKILL.md')), 'SKILL.md archived');
|
||||
assert.ok(existsSync(join(root, 'archive/skills/skill-b/references/cat3/b1.md')), 'ref archived');
|
||||
|
||||
// skills/skill-b is gone entirely.
|
||||
assert.equal(existsSync(join(skillsDir, 'skill-b')), false, 'retired skill dir removed');
|
||||
|
||||
// Sibling skill untouched.
|
||||
assert.ok(existsSync(join(skillsDir, 'skill-a/SKILL.md')), 'sibling skill survives');
|
||||
|
||||
assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'applied');
|
||||
});
|
||||
});
|
||||
|
||||
// ===========================================================================
|
||||
// (D) Safety — drift, preview, non-approved, guardrail-fail, unsupported op
|
||||
// ===========================================================================
|
||||
|
||||
test('applyApprovedAction — DRIFT after approval aborts with ZERO mutation and an untouched ledger', () => {
|
||||
withFixture(({ root, skillsDir, agentsDir, dataDir }) => {
|
||||
const approved = approveSanitize('skill-a', { skillsDir, agentsDir, dataDir });
|
||||
// Operator approved when there was 1 dead file. Now the disk drifts: the
|
||||
// dead file is already gone -> a fresh plan finds 0 dead -> guardrail differs.
|
||||
rmSync(join(skillsDir, 'skill-a/references/cat1/dead.md'));
|
||||
|
||||
const res = applyApprovedAction(approved, { pluginRoot: root, skillsDir, agentsDir, dataDir, apply: true });
|
||||
|
||||
assert.equal(res.applied, false, 'drift must abort the apply');
|
||||
assert.equal(res.revalidate.drift, true);
|
||||
assert.equal(existsSync(join(root, 'archive')), false, 'nothing archived on a drift abort');
|
||||
assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'approved', 'ledger left at approved');
|
||||
});
|
||||
});
|
||||
|
||||
test('applyApprovedAction — preview (apply:false) mutates nothing and leaves the ledger at approved', () => {
|
||||
withFixture(({ root, skillsDir, agentsDir, dataDir }) => {
|
||||
const approved = approveSanitize('skill-a', { skillsDir, agentsDir, dataDir });
|
||||
const res = applyApprovedAction(approved, { pluginRoot: root, skillsDir, agentsDir, dataDir, apply: false });
|
||||
|
||||
assert.equal(res.applied, false);
|
||||
assert.equal(res.preview, true);
|
||||
assert.equal(res.revalidate.ok, true, 'disk unchanged -> revalidation would pass');
|
||||
assert.ok(existsSync(join(skillsDir, 'skill-a/references/cat1/dead.md')), 'dead ref still present (preview only)');
|
||||
assert.equal(existsSync(join(root, 'archive')), false, 'preview archives nothing');
|
||||
assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'approved');
|
||||
});
|
||||
});
|
||||
|
||||
test('applyApprovedAction — refuses a non-approved (pending) entry with zero mutation', () => {
|
||||
withFixture(({ root, skillsDir, agentsDir, dataDir }) => {
|
||||
const { entry } = runSanitizePlan('skill-a', { skillsDir, agentsDir, dataDir, write: false });
|
||||
// pending, not approved — operator never signed off.
|
||||
saveDecisions(recordAction(loadDecisions(dataDir), entry), dataDir);
|
||||
|
||||
const res = applyApprovedAction(entry, { pluginRoot: root, skillsDir, agentsDir, dataDir, apply: true });
|
||||
assert.equal(res.applied, false);
|
||||
assert.match(res.revalidate.reason, /approved/i);
|
||||
assert.ok(existsSync(join(skillsDir, 'skill-a/references/cat1/dead.md')), 'no mutation');
|
||||
assert.equal(existsSync(join(root, 'archive')), false);
|
||||
});
|
||||
});
|
||||
|
||||
test('applyApprovedAction — a guardrail-failing approved entry never applies', () => {
|
||||
withFixture(({ root, skillsDir, agentsDir, dataDir }) => {
|
||||
// Construct an approved entry whose guardrail is NOT ok (illegal removal of a
|
||||
// curated ref). A fresh plan with the same removals also fails the guardrail.
|
||||
const { entry } = planSanitizeSkill('skill-a', {
|
||||
refs: ['cat1/live.md', 'cat1/dead.md'],
|
||||
orphans: ['cat1/dead.md'],
|
||||
removals: ['cat1/live.md'], // curated -> illegal
|
||||
});
|
||||
assert.equal(entry.guardrail.ok, false, 'precondition: this dry-run already failed the guardrail');
|
||||
const approved = { ...entry, status: 'approved' };
|
||||
saveDecisions(recordAction(loadDecisions(dataDir), approved), dataDir);
|
||||
|
||||
const res = applyApprovedAction(approved, { pluginRoot: root, skillsDir, agentsDir, dataDir, apply: true });
|
||||
assert.equal(res.applied, false, 'a failing guardrail must never reach the filesystem');
|
||||
assert.equal(existsSync(join(root, 'archive')), false);
|
||||
assert.ok(existsSync(join(skillsDir, 'skill-a/references/cat1/live.md')), 'curated ref still safe');
|
||||
});
|
||||
});
|
||||
|
||||
test('applyApprovedAction — merge_skills is unsupported this session (staged to S19) and throws', () => {
|
||||
withFixture(({ root, skillsDir, agentsDir, dataDir }) => {
|
||||
const approved = {
|
||||
operation_type: MERGE_OPERATION,
|
||||
status: 'approved',
|
||||
targets: { absorber: 'skill-a', absorbed: 'skill-b' },
|
||||
guardrail: { ok: true },
|
||||
};
|
||||
assert.throws(
|
||||
() => applyApprovedAction(approved, { pluginRoot: root, skillsDir, agentsDir, dataDir, apply: true }),
|
||||
/merge|unsupported|S19/i,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -21,6 +21,7 @@ import {
|
|||
isActionDecided,
|
||||
recordAction,
|
||||
listActions,
|
||||
setActionStatus,
|
||||
} from '../../scripts/kb-update/lib/decisions-io.mjs';
|
||||
|
||||
function withTmp(fn) {
|
||||
|
|
@ -154,3 +155,61 @@ test('saveDecisions + loadDecisions — actions round-trip through disk', () =>
|
|||
assert.deepEqual(back, led);
|
||||
});
|
||||
});
|
||||
|
||||
// --- setActionStatus — pure status transition (Sesjon 18: gates the apply-path) ---
|
||||
// The apply-path flips an operator-approved entry approved -> applied AFTER a
|
||||
// successful skills/ mutation. setActionStatus is the one pure primitive that
|
||||
// records that transition; it preserves every other field (targets, guardrail).
|
||||
|
||||
test('setActionStatus — transitions status, preserves targets+guardrail, is pure', () => {
|
||||
const led = recordAction(createLedger(), { ...MERGE, status: 'approved' });
|
||||
const key = actionKey(MERGE);
|
||||
const next = setActionStatus(led, key, 'applied', '2026-06-21');
|
||||
|
||||
assert.equal(led.actions[key].status, 'approved', 'input ledger untouched (pure)');
|
||||
assert.notEqual(next, led);
|
||||
assert.equal(next.actions[key].status, 'applied');
|
||||
assert.deepEqual(next.actions[key].targets, MERGE.targets, 'targets preserved');
|
||||
assert.deepEqual(next.actions[key].guardrail, MERGE.guardrail, 'guardrail preserved');
|
||||
});
|
||||
|
||||
test('setActionStatus — advances updated_at to the transition date', () => {
|
||||
const led = recordAction(createLedger(), { ...MERGE, status: 'approved' });
|
||||
const next = setActionStatus(led, actionKey(MERGE), 'applied', '2026-06-21');
|
||||
assert.equal(next.actions[actionKey(MERGE)].decided_at, '2026-06-21', 'entry date advances');
|
||||
assert.equal(next.updated_at, '2026-06-21');
|
||||
});
|
||||
|
||||
test('setActionStatus — without a date, keeps the entry decided_at and ledger updated_at', () => {
|
||||
let led = recordAction(createLedger(), { ...MERGE, status: 'approved' }); // decided_at 2026-06-20
|
||||
const next = setActionStatus(led, actionKey(MERGE), 'applied');
|
||||
assert.equal(next.actions[actionKey(MERGE)].decided_at, '2026-06-20', 'no date -> keep original');
|
||||
});
|
||||
|
||||
test('setActionStatus — throws on an unknown action key (cannot transition a non-entry)', () => {
|
||||
const led = recordAction(createLedger(), { ...MERGE, status: 'approved' });
|
||||
assert.throws(() => setActionStatus(led, 'sanitize_skill:ghost', 'applied'), /no action|ghost/i);
|
||||
});
|
||||
|
||||
test('setActionStatus — leaves the URL-keyed decisions map and sibling actions untouched', () => {
|
||||
let led = recordDecision(createLedger(), 'https://learn.microsoft.com/x', {
|
||||
status: 'approved',
|
||||
decided_at: '2026-06-19',
|
||||
});
|
||||
led = recordAction(led, { ...MERGE, status: 'approved' });
|
||||
led = recordAction(led, {
|
||||
operation_type: 'sanitize_skill',
|
||||
status: 'approved',
|
||||
decided_at: '2026-06-20',
|
||||
targets: { skill: 'ms-ai-governance' },
|
||||
guardrail: { ok: true },
|
||||
});
|
||||
const next = setActionStatus(led, actionKey(MERGE), 'applied', '2026-06-21');
|
||||
|
||||
assert.equal(isDecided(next, 'https://learn.microsoft.com/x'), true, 'URL decision survives');
|
||||
assert.equal(
|
||||
next.actions['sanitize_skill:ms-ai-governance'].status,
|
||||
'approved',
|
||||
'sibling action untouched',
|
||||
);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue