test(ultraplan-local): pin agent frontmatter contract (model + tools)
Pin the contract from plan-v2 Steps 1-3: every agents/*.md must declare model: (opus|sonnet|haiku) AND (tools: or disallowedTools:). Orchestrators (planning/research/review) must be opus and include the Agent tool; non-orchestrators must not include Agent (no recursive swarming). 23 agents in scope; 5 pinning tests. [skip-docs]
This commit is contained in:
parent
236be56ba5
commit
b1e161116a
1 changed files with 125 additions and 0 deletions
125
plugins/ultraplan-local/tests/lib/agent-frontmatter.test.mjs
Normal file
125
plugins/ultraplan-local/tests/lib/agent-frontmatter.test.mjs
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
// tests/lib/agent-frontmatter.test.mjs
|
||||
// Pin the agent-frontmatter contract from Steps 1-3 of plan-v2:
|
||||
// every agents/*.md MUST declare:
|
||||
// - model: (one of opus | sonnet | haiku)
|
||||
// - tools: (allowlist) OR disallowedTools: (denylist), at least one
|
||||
// Orchestrator agents (planning/research/review) MUST be model: opus and
|
||||
// MUST include the `Agent` tool in their tools allowlist (they spawn the swarm).
|
||||
//
|
||||
// When this test fails, fix the agent file — do NOT relax the assertion to
|
||||
// hide drift. The contract is what /ultra*-local commands rely on for
|
||||
// disciplined model selection + tool scoping.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { readFileSync, readdirSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const AGENTS_DIR = join(ROOT, 'agents');
|
||||
|
||||
const ORCHESTRATORS = new Set([
|
||||
'planning-orchestrator.md',
|
||||
'research-orchestrator.md',
|
||||
'review-orchestrator.md',
|
||||
]);
|
||||
|
||||
const ALLOWED_MODELS = new Set(['opus', 'sonnet', 'haiku']);
|
||||
|
||||
function read(rel) {
|
||||
return readFileSync(join(ROOT, rel), 'utf-8');
|
||||
}
|
||||
|
||||
function extractFrontmatter(text) {
|
||||
const m = text.match(/^---\r?\n([\s\S]*?)\r?\n---/);
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
function hasTopLevelKey(fm, key) {
|
||||
return new RegExp(`^${key}\\s*:`, 'm').test(fm);
|
||||
}
|
||||
|
||||
function getTopLevelValue(fm, key) {
|
||||
const m = fm.match(new RegExp(`^${key}\\s*:\\s*(.+?)\\s*$`, 'm'));
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
const agentFiles = readdirSync(AGENTS_DIR).filter(f => f.endsWith('.md'));
|
||||
|
||||
test('every agents/*.md declares a model: field', () => {
|
||||
assert.ok(agentFiles.length > 0, 'No agent files found under agents/');
|
||||
for (const f of agentFiles) {
|
||||
const fm = extractFrontmatter(read(`agents/${f}`));
|
||||
assert.ok(fm, `agents/${f}: missing YAML frontmatter block`);
|
||||
assert.ok(
|
||||
hasTopLevelKey(fm, 'model'),
|
||||
`agents/${f}: required \`model:\` field missing from frontmatter`,
|
||||
);
|
||||
const value = getTopLevelValue(fm, 'model');
|
||||
assert.ok(
|
||||
value && ALLOWED_MODELS.has(value),
|
||||
`agents/${f}: model: "${value}" must be one of ${[...ALLOWED_MODELS].join(' | ')}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('every agents/*.md declares tools: or disallowedTools:', () => {
|
||||
for (const f of agentFiles) {
|
||||
const fm = extractFrontmatter(read(`agents/${f}`));
|
||||
assert.ok(fm, `agents/${f}: missing YAML frontmatter block`);
|
||||
assert.ok(
|
||||
hasTopLevelKey(fm, 'tools') || hasTopLevelKey(fm, 'disallowedTools'),
|
||||
`agents/${f}: required \`tools:\` (allowlist) or \`disallowedTools:\` (denylist) field missing`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('every agents/*.md frontmatter name matches its filename', () => {
|
||||
for (const f of agentFiles) {
|
||||
const fm = extractFrontmatter(read(`agents/${f}`));
|
||||
assert.ok(fm, `agents/${f}: missing frontmatter`);
|
||||
const expected = f.replace(/\.md$/, '');
|
||||
const value = getTopLevelValue(fm, 'name');
|
||||
assert.equal(
|
||||
value,
|
||||
expected,
|
||||
`agents/${f}: frontmatter name="${value}" should match filename "${expected}"`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('orchestrator agents are model: opus and include the Agent tool', () => {
|
||||
for (const f of ORCHESTRATORS) {
|
||||
const path = `agents/${f}`;
|
||||
const fm = extractFrontmatter(read(path));
|
||||
assert.ok(fm, `${path}: missing frontmatter`);
|
||||
const model = getTopLevelValue(fm, 'model');
|
||||
assert.equal(
|
||||
model,
|
||||
'opus',
|
||||
`${path}: orchestrator must be model: opus (drives multi-agent swarm reasoning) — got "${model}"`,
|
||||
);
|
||||
const tools = getTopLevelValue(fm, 'tools');
|
||||
assert.ok(
|
||||
tools && /\bAgent\b/.test(tools),
|
||||
`${path}: orchestrator tools: must include "Agent" so it can spawn the swarm — got ${tools}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('non-orchestrator agents do NOT include the Agent tool (no recursive swarming)', () => {
|
||||
for (const f of agentFiles) {
|
||||
if (ORCHESTRATORS.has(f)) continue;
|
||||
const fm = extractFrontmatter(read(`agents/${f}`));
|
||||
assert.ok(fm, `agents/${f}: missing frontmatter`);
|
||||
const tools = getTopLevelValue(fm, 'tools');
|
||||
if (tools === null) continue; // disallowedTools-only agent — fine
|
||||
assert.ok(
|
||||
!/\bAgent\b/.test(tools),
|
||||
`agents/${f}: non-orchestrator must NOT include the Agent tool ` +
|
||||
`(only orchestrators spawn sub-agents) — got tools: ${tools}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue