voyage/tests/lib/skill-lint.test.mjs

// tests/lib/skill-lint.test.mjs
// The skill form check (S1): lib/skills/skill-lint.mjs. Model-free, part of
// `npm test`. Each rule is shown biting on a skill built for it (known
// positive) and passing a skill that keeps the form (known negative); the last
// tests run the check over the skills Voyage ships.

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { lintSkill, lintSkillsRoot, REQUIRED_HEADINGS, EXEMPTIONS } from '../../lib/skills/skill-lint.mjs';
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';

const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');
const CLI = join(ROOT, 'lib', 'skills', 'skill-lint.mjs');

const BODY = [
  '# Example', '',
  '## Overview', 'One line.', '',
  '## When to use', '- when the example applies', '- not when it does not', '',
  '## Process', '1. Do the thing.', '',
  '## Red flags', '- skipping the thing', '',
  '## Enforced by', '- prose only: nothing checks this.', '',
  '## Credits', '- Voyage\'s own content.', '',
].join('\n');

const GOOD_FM = [
  '---',
  'name: good-skill',
  'description: Checks an example the same way every time. Use when an example needs checking.',
  '---',
].join('\n');

/** A plugin-shaped temp root with skills/<name>/SKILL.md; returns the root. */
function plugin(skills) {
  const root = mkdtempSync(join(tmpdir(), 'skill-lint-'));
  for (const [name, text] of Object.entries(skills)) {
    mkdirSync(join(root, 'skills', name), { recursive: true });
    writeFileSync(join(root, 'skills', name, 'SKILL.md'), text);
  }
  return root;
}

const codes = (r) => r.errors.map((e) => e.code).sort();

function withSkill(name, text, fn) {
  const root = plugin({ [name]: text });
  try { return fn(lintSkill(join(root, 'skills', name), { pluginRoot: root }), root); }
  finally { rmSync(root, { recursive: true, force: true }); }
}

// ---- the test that fails first (ordre 7a) ----

test('skill-lint: a skill without "Use when" and with an unquoted colon is red', () => {
  const text = [
    '---',
    'name: bad-skill',
    'description: Checks things: all of them, every time.',
    '---',
    BODY,
  ].join('\n');
  withSkill('bad-skill', text, (r) => {
    assert.ok(codes(r).includes('SKILL_YAML_HAZARD'), codes(r).join(','));
    assert.ok(codes(r).includes('SKILL_DESC_NO_USE_WHEN'), codes(r).join(','));
  });
});

test('skill-lint: a skill that keeps the form is green (known negative)', () => {
  withSkill('good-skill', `${GOOD_FM}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
});

// ---- one rule at a time ----

test('skill-lint: name must equal the directory name and be kebab-case', () => {
  withSkill('other-dir', `${GOOD_FM}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_MISMATCH')));
  const fm = GOOD_FM.replace('name: good-skill', 'name: Good_Skill');
  withSkill('Good_Skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_FORM')));
  const reserved = GOOD_FM.replace('name: good-skill', 'name: claude-helper');
  withSkill('claude-helper', `${reserved}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_FORM')));
});

test('skill-lint: description is required, at most 1 024 characters', () => {
  const none = GOOD_FM.replace(/^description:.*$/m, 'license: MIT');
  withSkill('good-skill', `${none}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_MISSING')));
  const long = `Checks an example. Use when ${'an example needs checking and '.repeat(40)}done.`;
  assert.ok(long.length > 1024);
  const fm = GOOD_FM.replace(/^description:.*$/m, `description: ${long}`);
  withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_TOO_LONG')));
  const exact = `Use when x. ${'y'.repeat(1024 - 12)}`;
  assert.equal(exact.length, 1024);
  const fm2 = GOOD_FM.replace(/^description:.*$/m, `description: ${exact}`);
  withSkill('good-skill', `${fm2}\n${BODY}`, (r) => assert.ok(!codes(r).includes('SKILL_DESC_TOO_LONG')));
});

test('skill-lint: description is in the third person', () => {
  for (const d of [
    'I check an example. Use when an example needs checking.',
    'Checks your example. Use when an example needs checking.',
    'Use when you need an example checked.',
    'We check examples. Use when an example needs checking.',
  ]) {
    const fm = GOOD_FM.replace(/^description:.*$/m, `description: ${d}`);
    withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_NOT_THIRD_PERSON'), d));
  }
  const ok = GOOD_FM.replace(/^description:.*$/m, 'description: Checks an example, i.e. one case. Use when the user asks for a check.');
  withSkill('good-skill', `${ok}\n${BODY}`, (r) => assert.ok(!codes(r).includes('SKILL_DESC_NOT_THIRD_PERSON')));
});

test('skill-lint: YAML traps are caught: unquoted colon, comment marker, tab, unclosed quote, leading indicator', () => {
  for (const line of [
    'description: Use when x: y happens.',
    'description: Use when x happens #1.',
    'description:\tUse when x happens.',
    'description: "Use when x happens.',
    'description: `Use` when x happens.',
    'description: @Use when x happens.',
  ]) {
    const fm = GOOD_FM.replace(/^description:.*$/m, line);
    withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_YAML_HAZARD'), line));
  }
  // Quoted, the colon is safe.
  const quoted = GOOD_FM.replace(/^description:.*$/m, 'description: "Checks x: y. Use when x happens."');
  withSkill('good-skill', `${quoted}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
});

test('skill-lint: a multi-line block-scalar description is read whole', () => {
  const fm = ['---', 'name: good-skill', 'description: |', '  Checks an example the same way: every time.',
    '  Use when an example needs checking.', '---'].join('\n');
  withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
  const f = readFrontmatterFields(`${fm}\n${BODY}`);
  assert.equal(f.fields.description.value, 'Checks an example the same way: every time.\nUse when an example needs checking.\n');
  const folded = readFrontmatterFields(['---', 'description: >-', '  one', '  two', '---', ''].join('\n'));
  assert.equal(folded.fields.description.value, 'one two');
});

test('skill-lint: disable-model-invocation in any true spelling is refused', () => {
  for (const v of ['true', 'True', 'yes', 'on', '1']) {
    const fm = GOOD_FM.replace('---\nname', `---\ndisable-model-invocation: ${v}\nname`);
    withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_MODEL_INVOCATION_DISABLED'), v));
  }
  const off = GOOD_FM.replace('---\nname', '---\ndisable-model-invocation: false\nname');
  withSkill('good-skill', `${off}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
});

test('skill-lint: only Claude Code\'s own frontmatter fields are accepted', () => {
  const fm = GOOD_FM.replace('---\nname', '---\ntrigger: always\nname');
  withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_UNKNOWN_FIELD')));
  const known = GOOD_FM.replace('---\nname', '---\nlicense: MIT\nuser-invocable: true\nname');
  withSkill('good-skill', `${known}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
});

test('skill-lint: SKILL.md must stay under 500 lines', () => {
  const pad = Array.from({ length: 500 }, (_, i) => `- line ${i}`).join('\n');
  withSkill('good-skill', `${GOOD_FM}\n${BODY}\n${pad}\n`, (r) => assert.ok(codes(r).includes('SKILL_TOO_LONG')));
});

test('skill-lint: the fixed headings must all be there, in order', () => {
  assert.deepEqual(REQUIRED_HEADINGS, ['Overview', 'When to use', 'Process', 'Red flags', 'Enforced by', 'Credits']);
  withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('## Red flags', '## Warnings')}`,
    (r) => assert.ok(codes(r).includes('SKILL_MISSING_HEADING')));
  const swapped = BODY.replace('## Overview\nOne line.', '## Process\n1. x.').replace('## Process\n1. Do the thing.', '## Overview\nOne line.');
  withSkill('good-skill', `${GOOD_FM}\n${swapped}`, (r) => assert.ok(codes(r).includes('SKILL_HEADING_ORDER')));
  // A heading inside a fenced code block does not count.
  const fenced = BODY.replace('## Credits', '```\n## Credits\n```');
  withSkill('good-skill', `${GOOD_FM}\n${fenced}`, (r) => assert.ok(codes(r).includes('SKILL_MISSING_HEADING')));
});

test('skill-lint: "Enforced by" names a check that exists, or says prose only', () => {
  const none = BODY.replace('- prose only: nothing checks this.', '- something, somewhere.');
  withSkill('good-skill', `${GOOD_FM}\n${none}`, (r) => assert.ok(codes(r).includes('SKILL_ENFORCED_BY_VAGUE')));
  const real = BODY.replace('- prose only: nothing checks this.', '- `lib/check.mjs` stops it.');
  const root = plugin({ 'good-skill': `${GOOD_FM}\n${real}` });
  try {
    assert.ok(codes(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root })).includes('SKILL_REF_MISSING'));
    mkdirSync(join(root, 'lib'));
    writeFileSync(join(root, 'lib', 'check.mjs'), '');
    assert.deepEqual(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root }).errors, []);
  } finally { rmSync(root, { recursive: true, force: true }); }
});

test('skill-lint: files the skill points to must exist', () => {
  const linked = BODY.replace('One line.', 'One line. See [the detail](reference/detail.md).');
  const root = plugin({ 'good-skill': `${GOOD_FM}\n${linked}` });
  try {
    assert.ok(codes(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root })).includes('SKILL_REF_MISSING'));
    mkdirSync(join(root, 'skills', 'good-skill', 'reference'));
    writeFileSync(join(root, 'skills', 'good-skill', 'reference', 'detail.md'), '# detail\n');
    assert.deepEqual(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root }).errors, []);
  } finally { rmSync(root, { recursive: true, force: true }); }
});

test('skill-lint: paths under "Credits" name the source repository, not this plugin', () => {
  const credited = BODY.replace("- Voyage's own content.", '- owner/repo @abc1234, `skills/tdd/SKILL.md`: MIT.');
  withSkill('good-skill', `${GOOD_FM}\n${credited}`, (r) => assert.deepEqual(r.errors, []));
  // The same path anywhere else must exist.
  const elsewhere = BODY.replace('One line.', 'One line, see `skills/tdd/SKILL.md`.');
  withSkill('good-skill', `${GOOD_FM}\n${elsewhere}`, (r) => assert.ok(codes(r).includes('SKILL_REF_MISSING')));
});

test('skill-lint: no shell injection (inline or fenced), since Voyage advises turning it off', () => {
  withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', 'Status: !`git status`')}`,
    (r) => assert.ok(codes(r).includes('SKILL_SHELL_INJECTION')));
  withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', '```!\ngit status\n```')}`,
    (r) => assert.ok(codes(r).includes('SKILL_SHELL_INJECTION')));
  // A plain fenced bash block is fine; Claude reads it, nothing runs it on load.
  withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', '```bash\nnpm test\n```')}`,
    (r) => assert.deepEqual(r.errors, []));
});

test('skill-lint: an exemption lives in the check, never in the skill', () => {
  assert.ok(EXEMPTIONS instanceof Map);
  for (const [name, entry] of EXEMPTIONS) {
    assert.ok(Array.isArray(entry.codes) && entry.codes.length > 0, `${name}: exempt codes`);
    assert.ok(typeof entry.reason === 'string' && entry.reason.length > 10, `${name}: a reason`);
  }
  const self = `${GOOD_FM}\n<!-- skill-lint-exempt: SKILL_MISSING_HEADING -->\n${BODY}`;
  withSkill('good-skill', self, (r) => assert.ok(codes(r).includes('SKILL_SELF_EXEMPTION')));
  const meta = GOOD_FM.replace('---\nname', '---\nmetadata:\n  lint-exempt: true\nname');
  withSkill('good-skill', `${meta}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_SELF_EXEMPTION')));
});

test('skill-lint CLI: 0 clean, 1 findings, 2 could not run', () => {
  const good = plugin({ 'good-skill': `${GOOD_FM}\n${BODY}` });
  const bad = plugin({ 'bad-skill': `---\nname: bad-skill\ndescription: Checks: things.\n---\n${BODY}` });
  try {
    const g = spawnSync('node', [CLI, '--root', good], { encoding: 'utf8' });
    assert.equal(g.status, 0, g.stdout + g.stderr);
    const b = spawnSync('node', [CLI, '--root', bad, '--json'], { encoding: 'utf8' });
    assert.equal(b.status, 1, b.stdout + b.stderr);
    assert.ok(JSON.parse(b.stdout).skills['bad-skill'].errors.length > 0);
    const n = spawnSync('node', [CLI, '--root', join(good, 'nope')], { encoding: 'utf8' });
    assert.equal(n.status, 2, n.stdout + n.stderr);
  } finally {
    rmSync(good, { recursive: true, force: true });
    rmSync(bad, { recursive: true, force: true });
  }
});

// ---- the skills Voyage ships ----

test('skill-lint: every skill Voyage ships passes the form check', () => {
  const r = lintSkillsRoot(ROOT);
  assert.ok(Object.keys(r.skills).length >= 2, `found ${Object.keys(r.skills).join(',')}`);
  for (const [name, res] of Object.entries(r.skills)) {
    assert.deepEqual(res.errors, [], `${name}: ${JSON.stringify(res.errors, null, 2)}`);
  }
});