feat(ms-ai-architect): Sesjon 16 — B3 ledger action-lag + merge_skills dry-run-planner
Spor B / B3 start. Operatør-gated dry-run for skill-livssyklus, merge først.
- decisions-io.mjs: additivt skill-nøklet `actions`-lag ved siden av urørt
URL-nøklet `decisions` (14 URL-tester urørt = regresjonsbevis). version=1,
loadDecisions backfiller manglende actions-nøkkel. Nye rene fn: actionKey
(merge symmetrisk på paret), isActionDecided, recordAction (no-mutate),
listActions. Policy A gjelder også actions.
- scripts/kb-eval/lib/skill-ops.mjs: ren planMergeSkills → {entry,diff,guardrail},
leser ingen disk, anvender INGENTING. Guardrail = ingen kuratert verdi tapt
(count-invariant + set-equality + kollisjons-deteksjon; ok=false ved klobring).
Impur skall loadSkillRefs/runMergePlan + CLI plan-skill-op.mjs (gated --write).
- Ekte eng↔infra dry-run: guardrail OK (34+153=187, 0 kollisjoner), 153 fil-flytt
+ 7 taksonomi-reassign + retire eng; 0 skriving til skills/, ledger uendret
(retning er operatør/judge-valg, ikke pushet gjennom gaten).
- TDD: test-decisions-actions (10) + test-skill-ops-merge (10, inkl. byte-for-byte
skills/-snapshot som no-write-bevis). kb-eval 40→50, kb-update 122→132.
Regresjon grønn: validate 239, kb-integrity 192/192.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
ba597eb988
commit
f03a0e08b2
5 changed files with 672 additions and 4 deletions
174
tests/kb-eval/test-skill-ops-merge.test.mjs
Normal file
174
tests/kb-eval/test-skill-ops-merge.test.mjs
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
// tests/kb-eval/test-skill-ops-merge.test.mjs
|
||||
// Spor B / B3 (Sesjon 16): merge_skills as a GATED DRY-RUN PLANNER.
|
||||
//
|
||||
// Two layers:
|
||||
// (A) pure core — planMergeSkills(absorber, absorbed, ctx): mutates nothing,
|
||||
// reads no disk, returns { entry, diff, guardrail }. The guardrail proves
|
||||
// no curated value is lost (count-invariant + set-equality) and refuses
|
||||
// (ok=false) when two ref files would clobber on the same target path.
|
||||
// (B) impure shell — runMergePlan(...) + loadSkillRefs(...): read-only disk
|
||||
// load, optional gated `--write` of a PENDING entry to decisions.json.
|
||||
// The destructive proof: running the plan makes ZERO changes under skills/.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync, existsSync, readdirSync, statSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { planMergeSkills, loadSkillRefs, runMergePlan, MERGE_OPERATION } from '../../scripts/kb-eval/lib/skill-ops.mjs';
|
||||
import { loadDecisions, isActionDecided, actionKey } from '../../scripts/kb-update/lib/decisions-io.mjs';
|
||||
|
||||
// ===========================================================================
|
||||
// (A) Pure core — planMergeSkills
|
||||
// ===========================================================================
|
||||
|
||||
const CTX = {
|
||||
skillRefs: {
|
||||
'skill-a': ['cat1/a1.md', 'cat1/a2.md', 'cat2/a3.md'], // absorber, 3
|
||||
'skill-b': ['cat3/b1.md', 'cat3/b2.md'], // absorbed, 2
|
||||
},
|
||||
descriptions: { 'skill-a': 'desc A', 'skill-b': 'desc B' },
|
||||
categorySkill: { cat1: 'skill-a', cat2: 'skill-a', cat3: 'skill-b' },
|
||||
decided_at: '2026-06-20',
|
||||
};
|
||||
|
||||
test('planMergeSkills — guardrail OK for a clean (no-collision) union', () => {
|
||||
const { guardrail } = planMergeSkills('skill-a', 'skill-b', CTX);
|
||||
assert.equal(guardrail.ok, true);
|
||||
assert.equal(guardrail.preCountAbsorber, 3);
|
||||
assert.equal(guardrail.preCountAbsorbed, 2);
|
||||
assert.equal(guardrail.expectedPostCount, 5, 'no loss: union of both ref sets');
|
||||
assert.equal(guardrail.postCount, 5);
|
||||
assert.equal(guardrail.countInvariant, true);
|
||||
assert.equal(guardrail.setEquality, true);
|
||||
assert.deepEqual(guardrail.collisions, []);
|
||||
});
|
||||
|
||||
test('planMergeSkills — set-equality: EVERY ref from both skills survives in the plan', () => {
|
||||
const { diff, guardrail } = planMergeSkills('skill-a', 'skill-b', CTX);
|
||||
// absorbed files appear as moves; absorber files stay in place. Their union of
|
||||
// references-relative identities must equal the pre-merge union with no loss.
|
||||
const movedIdentities = diff.fileMoves.map((m) => m.to.replace('skills/skill-a/references/', ''));
|
||||
const survivingIdentities = new Set([...CTX.skillRefs['skill-a'], ...movedIdentities]);
|
||||
for (const r of [...CTX.skillRefs['skill-a'], ...CTX.skillRefs['skill-b']]) {
|
||||
assert.ok(survivingIdentities.has(r), `${r} must survive the merge`);
|
||||
}
|
||||
assert.equal(survivingIdentities.size, guardrail.expectedPostCount);
|
||||
});
|
||||
|
||||
test('planMergeSkills — guardrail FAILS (ok=false) when ref paths would collide/clobber', () => {
|
||||
const colliding = {
|
||||
...CTX,
|
||||
skillRefs: { 'skill-a': ['cat1/shared.md', 'cat1/a1.md'], 'skill-b': ['cat1/shared.md'] },
|
||||
};
|
||||
const { guardrail } = planMergeSkills('skill-a', 'skill-b', colliding);
|
||||
assert.equal(guardrail.ok, false, 'a clobber must not be silently approved');
|
||||
assert.deepEqual(guardrail.collisions, ['cat1/shared.md']);
|
||||
assert.equal(guardrail.countInvariant, false, 'postCount < expected -> loss detected');
|
||||
});
|
||||
|
||||
test('planMergeSkills — entry is a PENDING merge_skills action with both targets', () => {
|
||||
const { entry } = planMergeSkills('skill-a', 'skill-b', CTX);
|
||||
assert.equal(entry.operation_type, MERGE_OPERATION);
|
||||
assert.equal(entry.status, 'pending', 'planner only proposes — never approved');
|
||||
assert.deepEqual(entry.targets, { absorber: 'skill-a', absorbed: 'skill-b' });
|
||||
assert.equal(entry.decided_at, '2026-06-20');
|
||||
assert.equal(entry.guardrail.ok, true);
|
||||
});
|
||||
|
||||
test('planMergeSkills — diff carries moves, taxonomy reassignment, retire, and manual description reconciliation', () => {
|
||||
const { diff } = planMergeSkills('skill-a', 'skill-b', CTX);
|
||||
assert.deepEqual(diff.fileMoves, [
|
||||
{ from: 'skills/skill-b/references/cat3/b1.md', to: 'skills/skill-a/references/cat3/b1.md' },
|
||||
{ from: 'skills/skill-b/references/cat3/b2.md', to: 'skills/skill-a/references/cat3/b2.md' },
|
||||
]);
|
||||
assert.deepEqual(diff.taxonomyReassignments, [{ category: 'cat3', from: 'skill-b', to: 'skill-a' }]);
|
||||
assert.equal(diff.retire, 'skill-b');
|
||||
assert.equal(diff.descriptionReconciliation.absorber, 'desc A');
|
||||
assert.match(diff.descriptionReconciliation.note, /reconcil/i, 'semantic merge flagged as manual, not auto-done');
|
||||
});
|
||||
|
||||
test('planMergeSkills — throws on self-merge', () => {
|
||||
assert.throws(() => planMergeSkills('skill-a', 'skill-a', CTX), /self|same/i);
|
||||
});
|
||||
|
||||
test('planMergeSkills — missing ref entry defaults to empty (no throw)', () => {
|
||||
const { guardrail } = planMergeSkills('skill-a', 'ghost', { skillRefs: { 'skill-a': ['c/x.md'] } });
|
||||
assert.equal(guardrail.preCountAbsorbed, 0);
|
||||
assert.equal(guardrail.expectedPostCount, 1);
|
||||
assert.equal(guardrail.ok, true);
|
||||
});
|
||||
|
||||
// ===========================================================================
|
||||
// (B) Impure shell — loadSkillRefs + runMergePlan (read-only + gated ledger write)
|
||||
// ===========================================================================
|
||||
|
||||
// Build a throwaway skills/ fixture: two skills with a few non-colliding refs.
|
||||
function withSkillsFixture(fn) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'skillops-'));
|
||||
const mk = (skill, cat, file, body) => {
|
||||
const dir = join(root, 'skills', skill, 'references', cat);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(join(dir, file), body);
|
||||
};
|
||||
mk('skill-a', 'cat1', 'a1.md', '# A1\n');
|
||||
mk('skill-a', 'cat1', 'a2.md', '# A2\n');
|
||||
mk('skill-b', 'cat3', 'b1.md', '# B1\n');
|
||||
const dataDir = join(root, 'data');
|
||||
mkdirSync(dataDir, { recursive: true });
|
||||
try {
|
||||
return fn({ skillsDir: join(root, 'skills'), dataDir });
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot every file under a dir as path -> mtimeMs+size, for change detection.
|
||||
function snapshot(dir) {
|
||||
const out = {};
|
||||
const walk = (d) => {
|
||||
for (const e of readdirSync(d, { withFileTypes: true })) {
|
||||
const p = join(d, e.name);
|
||||
if (e.isDirectory()) walk(p);
|
||||
else { const s = statSync(p); out[p] = `${s.size}:${s.mtimeMs}`; }
|
||||
}
|
||||
};
|
||||
walk(dir);
|
||||
return out;
|
||||
}
|
||||
|
||||
test('loadSkillRefs — read-only, returns references-relative paths per skill', () => {
|
||||
withSkillsFixture(({ skillsDir }) => {
|
||||
const refs = loadSkillRefs(skillsDir);
|
||||
assert.deepEqual(refs['skill-a'].sort(), ['cat1/a1.md', 'cat1/a2.md']);
|
||||
assert.deepEqual(refs['skill-b'], ['cat3/b1.md']);
|
||||
});
|
||||
});
|
||||
|
||||
test('runMergePlan --write — records a PENDING entry in the ledger and touches NOTHING under skills/', () => {
|
||||
withSkillsFixture(({ skillsDir, dataDir }) => {
|
||||
const before = snapshot(skillsDir);
|
||||
const { entry, guardrail } = runMergePlan('skill-a', 'skill-b', { skillsDir, dataDir, write: true, decided_at: '2026-06-20' });
|
||||
|
||||
// proposal is correct
|
||||
assert.equal(entry.status, 'pending');
|
||||
assert.equal(guardrail.ok, true);
|
||||
assert.equal(guardrail.expectedPostCount, 3);
|
||||
|
||||
// ledger received the pending action (the gate's only write path)
|
||||
const led = loadDecisions(dataDir);
|
||||
assert.equal(isActionDecided(led, actionKey(entry)), true);
|
||||
assert.equal(led.actions[actionKey(entry)].status, 'pending');
|
||||
|
||||
// THE DESTRUCTIVE INVARIANT: skills/ is byte-for-byte unchanged
|
||||
assert.deepEqual(snapshot(skillsDir), before, 'dry-run must not write under skills/');
|
||||
});
|
||||
});
|
||||
|
||||
test('runMergePlan without --write — pure preview, no ledger file created', () => {
|
||||
withSkillsFixture(({ skillsDir, dataDir }) => {
|
||||
const { entry } = runMergePlan('skill-a', 'skill-b', { skillsDir, dataDir, write: false });
|
||||
assert.equal(entry.status, 'pending');
|
||||
assert.equal(existsSync(join(dataDir, 'decisions.json')), false, 'no write without --write');
|
||||
});
|
||||
});
|
||||
156
tests/kb-update/test-decisions-actions.test.mjs
Normal file
156
tests/kb-update/test-decisions-actions.test.mjs
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
// tests/kb-update/test-decisions-actions.test.mjs
|
||||
// Spor B / B3 (Sesjon 16): the ADDITIVE skill-level action layer on the lag-2
|
||||
// decision ledger. The URL-keyed `decisions` map (Spor A) is untouched — its
|
||||
// contract is covered by test-decisions-io.test.mjs, which stays green
|
||||
// unmodified, proving the extension is non-regressive. Here we cover only the
|
||||
// new `actions` collection: actionKey dedup, recordAction (pure), isActionDecided,
|
||||
// listActions, and backward-compatible normalization of pre-action ledgers.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
createLedger,
|
||||
loadDecisions,
|
||||
saveDecisions,
|
||||
recordDecision,
|
||||
isDecided,
|
||||
actionKey,
|
||||
isActionDecided,
|
||||
recordAction,
|
||||
listActions,
|
||||
} from '../../scripts/kb-update/lib/decisions-io.mjs';
|
||||
|
||||
function withTmp(fn) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'dec-act-test-'));
|
||||
try {
|
||||
return fn(dir);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
const MERGE = {
|
||||
operation_type: 'merge_skills',
|
||||
status: 'pending',
|
||||
decided_at: '2026-06-20',
|
||||
targets: { absorber: 'ms-ai-infrastructure', absorbed: 'ms-ai-engineering' },
|
||||
guardrail: { ok: true },
|
||||
note: 'dry-run',
|
||||
};
|
||||
|
||||
// --- scaffold now carries an empty actions collection (additive) ---
|
||||
|
||||
test('createLedger — additive actions:{} alongside untouched decisions:{}', () => {
|
||||
const led = createLedger();
|
||||
assert.equal(led.version, 1, 'version stays 1 — no migration needed');
|
||||
assert.deepEqual(led.decisions, {}, 'URL-keyed map unchanged');
|
||||
assert.deepEqual(led.actions, {}, 'new skill-op collection present and empty');
|
||||
});
|
||||
|
||||
// --- actionKey — deterministic dedup key ---
|
||||
|
||||
test('actionKey — merge_skills key is symmetric on the pair (direction-agnostic dedup)', () => {
|
||||
const k1 = actionKey(MERGE);
|
||||
const k2 = actionKey({
|
||||
...MERGE,
|
||||
targets: { absorber: 'ms-ai-engineering', absorbed: 'ms-ai-infrastructure' },
|
||||
});
|
||||
assert.equal(k1, k2, 'merging A<-B and B<-A must dedup to the same key');
|
||||
assert.equal(k1, 'merge_skills:ms-ai-engineering+ms-ai-infrastructure');
|
||||
});
|
||||
|
||||
test('actionKey — single-skill ops keyed by their one target', () => {
|
||||
assert.equal(
|
||||
actionKey({ operation_type: 'sanitize_skill', targets: { skill: 'ms-ai-governance' } }),
|
||||
'sanitize_skill:ms-ai-governance',
|
||||
);
|
||||
assert.equal(
|
||||
actionKey({ operation_type: 'create_skill', targets: { name: 'ms-power-platform' } }),
|
||||
'create_skill:ms-power-platform',
|
||||
);
|
||||
});
|
||||
|
||||
// --- recordAction — pure, mirrors recordDecision ---
|
||||
|
||||
test('recordAction — records under actionKey, returns new ledger, does not mutate input', () => {
|
||||
const led = createLedger();
|
||||
const next = recordAction(led, MERGE);
|
||||
assert.deepEqual(led.actions, {}, 'input untouched');
|
||||
assert.notEqual(next, led);
|
||||
assert.equal(next.actions['merge_skills:ms-ai-engineering+ms-ai-infrastructure'].status, 'pending');
|
||||
});
|
||||
|
||||
test('recordAction — advances updated_at to the action date', () => {
|
||||
const next = recordAction(createLedger(), MERGE);
|
||||
assert.equal(next.updated_at, '2026-06-20');
|
||||
});
|
||||
|
||||
test('recordAction — does NOT touch the URL-keyed decisions map', () => {
|
||||
let led = recordDecision(createLedger(), 'https://learn.microsoft.com/x', {
|
||||
status: 'approved',
|
||||
decided_at: '2026-06-19',
|
||||
});
|
||||
const next = recordAction(led, MERGE);
|
||||
assert.equal(isDecided(next, 'https://learn.microsoft.com/x'), true, 'URL decision survives');
|
||||
assert.equal(isActionDecided(next, actionKey(MERGE)), true, 'action recorded alongside');
|
||||
});
|
||||
|
||||
// --- isActionDecided — dedup gate (policy A: any status counts) ---
|
||||
|
||||
test('isActionDecided — true for ANY recorded status, false when absent', () => {
|
||||
let led = createLedger();
|
||||
assert.equal(isActionDecided(led, actionKey(MERGE)), false);
|
||||
led = recordAction(led, { ...MERGE, status: 'rejected' });
|
||||
assert.equal(isActionDecided(led, actionKey(MERGE)), true, 'rejected merge still dedups (policy A)');
|
||||
});
|
||||
|
||||
// --- listActions — flat list, optional status filter ---
|
||||
|
||||
test('listActions — returns all entries, filters by status when given', () => {
|
||||
let led = createLedger();
|
||||
led = recordAction(led, MERGE); // pending
|
||||
led = recordAction(led, {
|
||||
operation_type: 'sanitize_skill',
|
||||
status: 'approved',
|
||||
decided_at: '2026-06-20',
|
||||
targets: { skill: 'ms-ai-governance' },
|
||||
});
|
||||
assert.equal(listActions(led).length, 2);
|
||||
assert.deepEqual(
|
||||
listActions(led, 'pending').map((a) => a.operation_type),
|
||||
['merge_skills'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
listActions(led, 'approved').map((a) => a.operation_type),
|
||||
['sanitize_skill'],
|
||||
);
|
||||
});
|
||||
|
||||
// --- backward compatibility: a pre-action ledger on disk loads cleanly ---
|
||||
|
||||
test('loadDecisions — normalizes a pre-action (no actions key) ledger to actions:{}', () => {
|
||||
withTmp((dir) => {
|
||||
// Simulate an old on-disk ledger written before the action layer existed.
|
||||
writeFileSync(
|
||||
join(dir, 'decisions.json'),
|
||||
JSON.stringify({ version: 1, updated_at: null, decisions: {} }, null, 2),
|
||||
);
|
||||
const led = loadDecisions(dir);
|
||||
assert.deepEqual(led.actions, {}, 'missing actions key is backfilled, not undefined');
|
||||
// and recording an action on the loaded ledger works
|
||||
const next = recordAction(led, MERGE);
|
||||
assert.equal(isActionDecided(next, actionKey(MERGE)), true);
|
||||
});
|
||||
});
|
||||
|
||||
test('saveDecisions + loadDecisions — actions round-trip through disk', () => {
|
||||
withTmp((dir) => {
|
||||
const led = recordAction(createLedger(), MERGE);
|
||||
saveDecisions(led, dir);
|
||||
const back = loadDecisions(dir);
|
||||
assert.deepEqual(back, led);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue