voyage/tests/lib/skills-research-prose-pins.test.mjs

// tests/lib/skills-research-prose-pins.test.mjs
// Wave 2, order 7b: the `researching-sources` skill and its wiring.
//
// What is pinned, and why:
//   • the skill's form is not pinned here: order 7a's form check
//     (lib/skills/skill-lint.mjs) and routing test (lib/skills/routing.mjs)
//     run over every skill in `npm test` and are the authority (contract S1);
//   • the five rules, and the two measured red-baseline lines, stay in it;
//   • every web-search agent preloads the skill through its `skills:` line, and
//     /trekresearch names it where it starts them ("commands are the trunk");
//   • the shared source rules that moved into the skill are gone from the
//     agents — including the two sentences that contradicted it (an empty
//     counter-evidence search "increases confidence"; fall back "silently");
//   • one confidence scale: the skill's table and /trekresearch triangulation
//     carry the same four definitions;
//   • the skill says plainly what enforces it and that the rest is text only.
//
// The behaviour itself (reporting an injected instruction; not counting an
// empty search as support) was measured with helpers in the authoring session
// and is not something a model-free test can see. The eval cases under
// evals/skill-researching-sources-* carry it for the operator's own runs.

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { readFileSync, readdirSync, existsSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';

const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');
const read = (rel) => readFileSync(join(ROOT, rel), 'utf-8');

const SKILL = 'skills/researching-sources/SKILL.md';
const WEB_AGENTS = [
  'docs-researcher',
  'community-researcher',
  'security-researcher',
  'contrarian-researcher',
  'research-scout',
];

function frontmatter(text) {
  const m = text.match(/^---\n([\s\S]*?)\n---\n/);
  assert.ok(m, 'file must open with a frontmatter block');
  return m[1];
}

function field(fm, key) {
  const m = fm.match(new RegExp(`^${key}:\\s*(.*)$`, 'm'));
  return m ? m[1].trim() : null;
}

function listField(fm, key) {
  const raw = field(fm, key);
  if (raw === null) return [];
  if (raw.startsWith('[')) return JSON.parse(raw);
  return [raw];
}

function section(text, startRe, endRe) {
  const start = text.search(startRe);
  assert.ok(start >= 0, `section ${startRe} not found`);
  const rest = text.slice(start);
  const end = rest.slice(1).search(endRe);
  return end < 0 ? rest : rest.slice(0, end + 1);
}

// --- the skill's content ----------------------------------------------------

test('researching-sources: carries the five rules the order names', () => {
  const text = read(SKILL);
  for (const re of [
    /official sources? first/i,
    /no citation, no finding/i,
    /absence of data is not a finding/i,
    /data, never instructions/i,
    /outbound queries carry nothing local/i,
  ]) assert.match(text, re, `rule missing: ${re}`);
  assert.match(text, /\*\*Report it\.\*\*/, 'an injected instruction is reported, not only ignored (the measured red baseline)');
  assert.match(text, /never raises a\s+confidence rating/i, 'an empty search never raises a rating (the measured red baseline)');
});

test('researching-sources: "Enforced by" names the structure-only check and says the rest is prose only', () => {
  const enforced = section(read(SKILL), /^## Enforced by/m, /^## /m);
  assert.match(enforced, /research-validator\.mjs/);
  assert.match(enforced, /structure/i);
  assert.match(enforced, /query-privacy-gate\.mjs/);
  assert.match(enforced, /off by default/i, 'the privacy gate runs only in the default-off steps; say so');
  assert.match(enforced, /\*\*prose only\*\*/);
});

test('researching-sources: credits carry source, commit and licence', () => {
  const credits = section(read(SKILL), /^## Credits/m, /^## /m);
  assert.match(credits, /source-driven-development/);
  assert.match(credits, /9d0c60d/);
  assert.match(credits, /SKILL\.md:97-114/);
  assert.match(credits, /diagnosing-superpowers/);
  assert.match(credits, /8ca22db/);
  assert.equal((credits.match(/\bMIT\b/g) || []).length, 2, 'both borrowed sources are MIT and say so');
});

// --- wiring: agents preload it, the command names it -----------------------

test('every web-search agent preloads researching-sources through its skills: line', () => {
  for (const a of WEB_AGENTS) {
    const fm = frontmatter(read(`agents/${a}.md`));
    const skills = listField(fm, 'skills');
    assert.ok(
      skills.some((s) => s === 'voyage:researching-sources' || s === 'researching-sources'),
      `agents/${a}.md skills: must list researching-sources (got ${JSON.stringify(skills)})`,
    );
  }
});

test('the web-search agent list is complete: every agent with WebSearch or WebFetch is in it', () => {
  const withWeb = readdirSync(join(ROOT, 'agents'))
    .filter((f) => f.endsWith('.md'))
    .filter((f) => /^tools:.*\bWeb(Search|Fetch)\b/m.test(frontmatter(read(`agents/${f}`))))
    .map((f) => f.replace(/\.md$/, ''))
    .sort();
  assert.deepEqual(withWeb, [...WEB_AGENTS].sort());
});

test('/trekresearch names researching-sources where it starts the external agents', () => {
  const ext = section(read('commands/trekresearch.md'), /^### External agents/m, /^### Launch rules/m);
  assert.match(ext, /`researching-sources` skill/);
  assert.match(ext, /voyage:researching-sources/);
  assert.match(ext, /Name it in every external spawn prompt/);
});

// --- the copied text is gone ------------------------------------------------

// Measured 08.10 on 4ee711a: these sentences carried the shared source rules in
// the agents. Each now lives in the skill; a copy coming back is drift.
const MOVED = [
  ['docs-researcher', 'Never invent documentation.'],
  ['docs-researcher', 'Always include source URLs.'],
  ['docs-researcher', 'Date everything.'],
  ['docs-researcher', 'Flag version mismatches.'],
  ['docs-researcher', 'Fall back to WebSearch silently'],
  ['research-scout', 'Never invent documentation.'],
  ['research-scout', 'Always include source URLs.'],
  ['research-scout', 'Date everything.'],
  ['research-scout', 'Fall back to WebSearch silently'],
  ['community-researcher', 'Date your sources.'],
  ['community-researcher', 'Flag if a "problem" has since been fixed.'],
  ['security-researcher', 'Distinguish absence of data from absence of vulnerabilities.'],
  ['security-researcher', 'Every security concern raised must have a verifiable source.'],
  ['contrarian-researcher', 'that IS a finding'],
  ['contrarian-researcher', 'This increases confidence in the original conclusion'],
  ['contrarian-researcher', 'it is a valuable finding'],
  ['contrarian-researcher', 'Every counter-argument needs a real source.'],
  ['contrarian-researcher', 'Check recency.'],
];

test('the source rules that moved into the skill are no longer copied into the agents', () => {
  const back = MOVED.filter(([a, s]) => read(`agents/${a}.md`).includes(s));
  assert.deepEqual(back, [], 'copied rule text is back in an agent — it belongs to the skill');
});

test('contrarian-researcher: an empty counter-evidence search is reported, and is not support', () => {
  const text = read('agents/contrarian-researcher.md');
  assert.match(text, /absence of counter-evidence is not support/);
  assert.match(text, /does not raise\s+confidence in the original conclusion/);
});

test('each web-search agent keeps its own role and points at the skill for the shared rules', () => {
  for (const a of WEB_AGENTS) {
    const body = read(`agents/${a}.md`).replace(/^---\n[\s\S]*?\n---\n/, '');
    assert.match(body, /preloaded\s+`researching-sources` skill/, `agents/${a}.md must point at the preloaded skill`);
    assert.match(body, /This\s+file holds only your own role\./, `agents/${a}.md must say the rest is its own role`);
  }
});

// --- one confidence scale ---------------------------------------------------

function norm(s) {
  return s.replace(/\*\*/g, '').replace(/\s+/g, ' ').trim().replace(/\.$/, '').toLowerCase();
}

test('one confidence scale: the skill table and /trekresearch triangulation define the same four ratings', () => {
  const table = section(read(SKILL), /^### Confidence scale/m, /\n##+ /);
  const skillRows = [...table.matchAll(/^\| \*\*(\w+)\*\* \| (.+) \|$/gm)].map((m) => [m[1], norm(m[2])]);
  const tri = section(read('commands/trekresearch.md'), /^## Phase 6 — Triangulation/m, /^## Phase 7/m);
  const cmdRows = [...tri.matchAll(/^ {3}- \*\*(\w+)\*\* — (.+)$/gm)].map((m) => [m[1], norm(m[2])]);
  assert.deepEqual(skillRows.map((r) => r[0]), ['high', 'medium', 'low', 'contradictory']);
  assert.deepEqual(cmdRows, skillRows, 'Phase 6 must define the ratings word for word as the skill does');
});

test('one confidence scale: the deep-research adapter and the brief template accept the same four ratings', () => {
  const cmd = read('commands/trekresearch.md');
  assert.match(cmd, /`### \{Dimension\} -- Confidence: \{high\|medium\|low\|contradictory\}`/);
  assert.match(read('templates/research-brief-template.md'), /Confidence: \{high \| medium \| low \| contradictory\}/);
});

// --- the eval cases ---------------------------------------------------------

test('researching-sources eval cases: 3 should trigger, 2 should not, 1 effect case', () => {
  const dirs = readdirSync(join(ROOT, 'evals')).filter((d) => d.startsWith('skill-researching-sources-'));
  const hits = dirs.filter((d) => d.includes('-hit-'));
  const misses = dirs.filter((d) => d.includes('-miss-'));
  const effect = dirs.filter((d) => d.includes('-effect-'));
  assert.ok(hits.length >= 3, `need >= 3 should-trigger cases, have ${hits.length}`);
  assert.ok(misses.length >= 2, `need >= 2 should-not-trigger cases, have ${misses.length}`);
  assert.equal(effect.length, 1);
  for (const d of hits) assert.match(read(`evals/${d}/graders/skill-loaded.md`), /tool: Skill[\s\S]*min: 1/);
  for (const d of misses) assert.match(read(`evals/${d}/graders/skill-not-loaded.md`), /min: 0\nmax: 0/);
  for (const d of dirs) {
    assert.ok(existsSync(join(ROOT, 'evals', d, 'prompt.md')), `${d}/prompt.md`);
    assert.doesNotMatch(read(`evals/${d}/prompt.md`), /\bproevesett\b/, 'skill cases are not part of the fixed test set');
  }
});