/** * R4 — an agent file must not carry a contract its `tools:` cannot keep. * * `agents/verifier-agent.md` said both things at once: §Output Format * ("Append to: implementation-log.md") and §Read-Only Guarantee ("never * modifies any files"), while its frontmatter granted only Read/Glob/Grep. * Which instruction wins is nondeterministic, and the failure mode is not * "the write fails" — it is the agent improvising a full-file Write on the * SHARED implementation log, clobbering parallel implementer entries. That is * exactly the defect `implement-log-append.test.mjs` exists to prevent, * entering through the one file that test does not read. * * The guard is the blanket invariant over the whole agents/ catalogue, not a * statement about verifier-agent: any agent whose tools grant no write * capability must (a) instruct no file write, and (b) say positively that it * returns its findings inline. Both sides are read from the file — the tools * list AND the body — so removing a write tool from any agent whose body still * instructs a write turns this red. */ import { test } from 'node:test'; import { strict as assert } from 'node:assert'; import { readFile, readdir } from 'node:fs/promises'; import { resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; const __dirname = dirname(fileURLToPath(import.meta.url)); const ROOT = resolve(__dirname, '..', '..'); const AGENTS_DIR = resolve(ROOT, 'agents'); /** Tools that can put bytes on disk. Bash counts: `>>` is a write. */ const WRITE_TOOLS = new Set(['Write', 'Edit', 'NotebookEdit', 'Bash']); /** * Directive lines that tell the agent to put its output in a file. * Anchored to line start so prose ABOUT writing ("do not write it to a file") * is not caught — the defect is an instruction, not a mention. */ const WRITE_DIRECTIVE_RE = /^(?:\*\*)?(?:Append|Write|Save|Output|Persist)\b(?![^\n]*\bnot\b)[^\n]*?(?:\bto\b|`[^`\n]+\.(?:md|ya?ml|json)`)/mi; /** The positive half: the file must say the findings come back inline. */ const RETURN_INLINE_RE = /\breturn\b[^.]{0,120}?\b(?:as\s+your\s+final\s+message|inline)\b/i; function frontmatterOf(content) { const m = content.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n/); return m ? m[1] : ''; } function bodyOf(content) { const m = content.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n/); return m ? content.slice(m[0].length) : content; } function toolsOf(frontmatter) { const m = frontmatter.match(/^tools:\s*(.+)$/m); if (!m) return []; return (m[1].match(/[A-Za-z_][A-Za-z0-9_]*/g) || []); } async function loadAgents() { const names = (await readdir(AGENTS_DIR)).filter((n) => n.endsWith('.md')).sort(); return Promise.all( names.map(async (name) => { const content = await readFile(resolve(AGENTS_DIR, name), 'utf-8'); const frontmatter = frontmatterOf(content); const tools = toolsOf(frontmatter); return { name, body: bodyOf(content), tools, canWrite: tools.some((t) => WRITE_TOOLS.has(t)), }; }), ); } test('R4 sweep is not vacuous: the agents catalogue is read and at least one agent has no write tool', async () => { const agents = await loadAgents(); assert.ok(agents.length >= 7, `expected the agents/ catalogue to be swept, got ${agents.length} files`); assert.ok(agents.every((a) => a.tools.length > 0), `every agent must declare tools:, missing in ${agents.filter((a) => !a.tools.length).map((a) => a.name).join(', ')}`); const writeless = agents.filter((a) => !a.canWrite); assert.ok(writeless.length >= 1, 'no write-tool-less agent found — the invariant below would be vacuously green'); }); test('R4: no agent instructs a file write its tools cannot perform', async () => { const agents = await loadAgents(); for (const agent of agents.filter((a) => !a.canWrite)) { const offending = agent.body.match(WRITE_DIRECTIVE_RE); assert.ok( offending === null, `${agent.name} grants no write tool (tools: ${agent.tools.join(', ')}) but instructs a write: ${JSON.stringify(offending && offending[0])}`, ); } }); test('R4: a write-tool-less agent states positively that it returns findings inline', async () => { const agents = await loadAgents(); for (const agent of agents.filter((a) => !a.canWrite)) { assert.ok( RETURN_INLINE_RE.test(agent.body), `${agent.name} has no write tool, so it must say its findings are returned as its final message`, ); } });