merge: main (7d, 7c) into 7e
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
0db3fe4be3
49 changed files with 1206 additions and 120 deletions
4
tests/fixtures/jsonl-schemas.md
vendored
4
tests/fixtures/jsonl-schemas.md
vendored
|
|
@ -21,11 +21,11 @@
|
|||
|-----------|--------|-------------|----------|---------------|-----|
|
||||
| trekbrief-stats | ts, task, slug, mode, interview_turns, review_iterations, brief_quality, research_topics, auto_research, auto_result, intent_approved, project_dir, order_id, order_claimed_at, order_reason | commands/trekbrief.md (orchestrator-emit Phase 7; order_* added by lib/stats/stats-append.mjs) | trekbrief.md:1127-1151 | profile, phase_models, profile_source | none |
|
||||
| trekresearch-stats | ts, question, mode, scope, engine, slug, project_dir, brief_path, dimensions, dimensions_baseline, dimensions_baseline_preserved, effort, conv_turns, empty_turns, unique_sources, agents_local, agents_external, gemini_used, confidence, contradictions, open_questions | lib/stats/stats-append.mjs (trekresearch.md Stats tracking) | trekresearch.md:782-808 | profile, phase_models, parallel_agents, external_research_enabled, profile_source | none |
|
||||
| trekplan-stats | ts, task, mode, slug, brief_path, project_dir, codebase_size, codebase_files, agents_deployed, deep_dives, research_briefs_used, research_scout_used, critic_verdict, guardian_verdict, outcome | lib/stats/stats-append.mjs (trekplan.md Phase 12) | trekplan.md:1271-1299 | profile, phase_models, parallel_agents, profile_source | none |
|
||||
| trekplan-stats | ts, task, mode, slug, brief_path, project_dir, codebase_size, codebase_files, agents_deployed, deep_dives, research_briefs_used, research_scout_used, critic_verdict, guardian_verdict, outcome | lib/stats/stats-append.mjs (trekplan.md Phase 12) | trekplan.md:1277-1305 | profile, phase_models, parallel_agents, profile_source | none |
|
||||
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2892-2914 | profile, phase_models, profile_source | none |
|
||||
| trekexecute-stats (autonomy events) | ts, event, known_event, payload | lib/stats/event-emit.mjs `emit()` | event-emit.mjs:78-99 | payload.profile, payload.phase_models, payload.profile_source | none |
|
||||
| trekexecute-stats (PostToolUse Bash) | ts, session_id, command_excerpt, duration_ms, success | hooks/scripts/post-bash-stats.mjs (Bash PostToolUse) | post-bash-stats.mjs:42-54 | none (hook is plugin-level, not profile-aware) | command_excerpt (CWE-212) |
|
||||
| trekreview-stats | ts, slug, verdict, counts (BLOCKER/MAJOR/MINOR/SUGGESTION), reviewed_files_count, mode, duration_ms | lib/stats/stats-append.mjs (trekreview.md Phase 8) | trekreview.md:623-673 | profile, phase_models, profile_source | none |
|
||||
| trekreview-stats | ts, slug, verdict, counts (BLOCKER/MAJOR/MINOR/SUGGESTION), reviewed_files_count, mode, duration_ms | lib/stats/stats-append.mjs (trekreview.md Phase 8) | trekreview.md:636-686 | profile, phase_models, profile_source | none |
|
||||
| trekcontinue-stats | ts, project, next_session_label, status | lib/stats/stats-append.mjs (trekcontinue.md Phase 5) | trekcontinue.md:293-297 | profile, profile_source | none |
|
||||
| token-usage | ts, session_id, scope, model, tokens_input, tokens_output, tokens_cache_creation, tokens_cache_read, cost_usd, is_estimate, price_table_version | lib/stats/token-usage.mjs `captureTokenUsage` (via hooks/scripts/otel-export.mjs Stop hook, opt-in `VOYAGE_TOKEN_METER`) | token-usage.mjs:buildRecord | n/a (SKAL-2 schema) | session_id (stripped at export) |
|
||||
|
||||
|
|
|
|||
|
|
@ -873,7 +873,9 @@ test('review 630 r7 / brief note 3: the prose says a git revision may start with
|
|||
const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
||||
assert.equal(allowedCommand('git show @~1 --stat', { cwd: ROOT }).allowed, true);
|
||||
assert.equal(allowedCommand('pytest @args.txt', { cwd: ROOT }).allowed, false, 'the control: a runner word with a leading @');
|
||||
for (const rel of ['agents/brief-reviewer.md', 'commands/trekplan.md']) {
|
||||
// The runner's rules for writing a criterion moved from brief-reviewer and
|
||||
// /trekplan into the writing-criteria skill (order 7c, 17.10).
|
||||
for (const rel of ['skills/writing-criteria/SKILL.md']) {
|
||||
assert.match(read(rel), /`@~1`/, `${rel} names the git revision as the exception`);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
227
tests/lib/skills-criteria-prose-pins.test.mjs
Normal file
227
tests/lib/skills-criteria-prose-pins.test.mjs
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
// tests/lib/skills-criteria-prose-pins.test.mjs
|
||||
// Wave 3, order 7c: the `writing-criteria` skill and its wiring (contract S3,
|
||||
// and S4 on the two agents order 7c owns).
|
||||
//
|
||||
// What is pinned, and why:
|
||||
// • the skill's form is not pinned here: the form check
|
||||
// (lib/skills/skill-lint.mjs) and the routing test (lib/skills/routing.mjs)
|
||||
// run over every skill in `npm test` and are the authority (contract S1);
|
||||
// • every example the skill marks `Runs:` is accepted by the criteria
|
||||
// runner, every `Refused:` is refused, and every `Never red:` is accepted
|
||||
// (the runner cannot see that it never fails; the skill says so) — so the
|
||||
// skill can never teach a criterion the runner refuses;
|
||||
// • brief-reviewer and plan-critic preload both review skills, and
|
||||
// /trekplan names writing-criteria where the plan is written and both
|
||||
// skills where it starts its three reviewers;
|
||||
// • the criteria text the skill replaced is gone from the agents, the
|
||||
// templates and /trekplan (measured 17.10 on 31c77fd; one copy left);
|
||||
// • every runnable example in the two templates is accepted by the runner.
|
||||
//
|
||||
// The behaviour itself (writing runnable criteria; a reviewer naming one that
|
||||
// cannot go red) was measured with helpers in the authoring session and is not
|
||||
// something a model-free test can see. The eval cases under
|
||||
// evals/skill-writing-criteria-* carry it for the operator's own runs.
|
||||
// Counted as doc pins by lib/util/test-census.mjs (file name *-prose-pins).
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { existsSync, readdirSync, readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { firstCommand, runnableCheck } from '../../lib/verification/criteria-runner.mjs';
|
||||
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const read = (rel) => readFileSync(join(ROOT, rel), 'utf8');
|
||||
|
||||
const SKILL = 'skills/writing-criteria/SKILL.md';
|
||||
const BOTH = ['voyage:writing-criteria', 'voyage:reviewing-adversarially'];
|
||||
|
||||
function section(text, startRe, endRe) {
|
||||
const start = text.search(startRe);
|
||||
assert.ok(start >= 0, `section ${startRe} not found`);
|
||||
const rest = text.slice(start);
|
||||
const end = rest.slice(1).search(endRe);
|
||||
return end < 0 ? rest : rest.slice(0, end + 1);
|
||||
}
|
||||
|
||||
const runs = (span) => runnableCheck(firstCommand(`\`${span}\``), { cwd: ROOT }).runnable;
|
||||
|
||||
/**
|
||||
* The examples a text marks, and each one that the runner judges otherwise
|
||||
* than its mark says. `Runs:` and `Never red:` must be accepted, `Refused:`
|
||||
* refused.
|
||||
*/
|
||||
export function markedExamples(text) {
|
||||
const out = [];
|
||||
for (const m of text.matchAll(/^- (Runs|Refused|Never red): `([^`]+)`/gm)) {
|
||||
const runnable = runs(m[2]);
|
||||
const want = m[1] !== 'Refused';
|
||||
out.push({ mark: m[1], command: m[2], runnable, wrong: runnable !== want });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- the skill's examples go through the criteria runner ---------------------
|
||||
|
||||
test('writing-criteria: every Runs:/Never red: example is accepted and every Refused: example refused', () => {
|
||||
const ex = markedExamples(read(SKILL));
|
||||
const count = (mark) => ex.filter((e) => e.mark === mark).length;
|
||||
assert.ok(count('Runs') >= 5, `need >= 5 Runs: examples, have ${count('Runs')}`);
|
||||
assert.ok(count('Refused') >= 5, `need >= 5 Refused: examples, have ${count('Refused')}`);
|
||||
assert.ok(count('Never red') >= 1, 'the skill shows at least one runnable check that can never go red');
|
||||
assert.deepEqual(ex.filter((e) => e.wrong), [], 'an example marked otherwise than the runner judges it');
|
||||
});
|
||||
|
||||
test('writing-criteria: the example check is red on a stub that marks a pipe as runnable', () => {
|
||||
const stub = '- Runs: `npm test`\n- Runs: `node bin/app.mjs list --json | jq length`\n- Refused: `npm test && npm run lint`\n';
|
||||
const wrong = markedExamples(stub).filter((e) => e.wrong).map((e) => e.command);
|
||||
assert.deepEqual(wrong, ['node bin/app.mjs list --json | jq length']);
|
||||
// And the other way round: a runnable command marked refused is caught too.
|
||||
assert.deepEqual(markedExamples('- Refused: `npm test`\n').filter((e) => e.wrong).length, 1);
|
||||
});
|
||||
|
||||
test('writing-criteria: the measured red-baseline failures are named in the skill', () => {
|
||||
const text = read(SKILL);
|
||||
assert.match(text, /\*\*One criterion, one command\.\*\*/);
|
||||
assert.match(text, /\*\*The command is the first backticked span on the line\.\*\*/);
|
||||
assert.match(text, /\*\*A check that needs more is a script\.\*\*/);
|
||||
assert.match(text, /\*\*It must be able to go red\.\*\*/);
|
||||
assert.match(text, /`bash tests\/<name>\.sh`/);
|
||||
});
|
||||
|
||||
test('writing-criteria: "Enforced by" names the runner and the validators, and says go-red is prose only', () => {
|
||||
const enforced = section(read(SKILL), /^## Enforced by/m, /^## /m);
|
||||
assert.match(enforced, /lib\/verification\/criteria-runner\.mjs/);
|
||||
assert.match(enforced, /deterministic/);
|
||||
assert.match(enforced, /lib\/validators\/plan-validator\.mjs/);
|
||||
assert.match(enforced, /lib\/validators\/brief-validator\.mjs/);
|
||||
assert.match(enforced, /prose only: that a criterion can go red and that it tests what the brief asks for/);
|
||||
assert.match(enforced, /`brief-reviewer`\s+and `plan-critic` judge that by reading/);
|
||||
});
|
||||
|
||||
test('writing-criteria: credits say it is Voyage\'s own content, nothing borrowed', () => {
|
||||
const credits = section(read(SKILL), /^## Credits/m, /^## /m);
|
||||
assert.match(credits, /Voyage's own content; nothing is borrowed/);
|
||||
assert.doesNotMatch(credits, /\bMIT\b|@[0-9a-f]{7}/);
|
||||
});
|
||||
|
||||
// --- wiring: two agents preload it, /trekplan names it -----------------------
|
||||
|
||||
test('brief-reviewer and plan-critic preload writing-criteria and reviewing-adversarially (S3 + S4), tools unchanged', () => {
|
||||
for (const a of ['brief-reviewer', 'plan-critic']) {
|
||||
const fm = readFrontmatterFields(read(`agents/${a}.md`)).fields;
|
||||
assert.deepEqual(fm.skills?.value, BOTH, `agents/${a}.md skills:`);
|
||||
assert.deepEqual(fm.tools.value, ['Read', 'Glob', 'Grep'], `agents/${a}.md keeps its read-only tools`);
|
||||
}
|
||||
});
|
||||
|
||||
test('/trekplan names writing-criteria where the plan is written', () => {
|
||||
const p8 = section(read('commands/trekplan.md'), /^## Phase 8 — Deep planning/m, /^## Phase 9/m);
|
||||
assert.match(p8, /voyage:writing-criteria/);
|
||||
});
|
||||
|
||||
test('/trekplan names both skills where it starts brief-reviewer, plan-critic and scope-guardian', () => {
|
||||
const cmd = read('commands/trekplan.md');
|
||||
const p4b = section(cmd, /^## Phase 4b — Brief review/m, /^## Phase 5/m);
|
||||
const p9 = section(cmd, /^Both reviewers are asked for/m, /^After both complete/m);
|
||||
// The plugin-qualified names stand where the agents are started ...
|
||||
for (const [name, text] of [['brief-reviewer', p4b], ['plan-critic and scope-guardian', p9]]) {
|
||||
for (const s of BOTH) assert.ok(text.includes(s), `/trekplan must name ${s} where it starts ${name}`);
|
||||
}
|
||||
// ... and each of the three prompts asks for both skills itself.
|
||||
const prompts = [
|
||||
['brief-reviewer', p4b.split('Prompt:')[1]],
|
||||
['plan-critic', p9.split('**plan-critic**')[1].split('**scope-guardian**')[0]],
|
||||
['scope-guardian', p9.split('**scope-guardian**')[1]],
|
||||
];
|
||||
for (const [name, text] of prompts) {
|
||||
assert.match(text, /Follow the writing-criteria skill[^(]*\([^)]*go\s+red[^)]*\)/, `${name}: its prompt asks for writing-criteria, with the rule`);
|
||||
assert.match(text, /the\s+reviewing-adversarially\s+skill/, `${name}: its prompt asks for reviewing-adversarially`);
|
||||
}
|
||||
});
|
||||
|
||||
// --- the copied text is gone ------------------------------------------------
|
||||
|
||||
// Measured 17.10 on 31c77fd: these sentences carried the runner's rules for
|
||||
// writing a criterion. Each now lives in the skill; a copy coming back is drift.
|
||||
const MOVED = [
|
||||
['agents/brief-reviewer.md', '**Runnable at review.** `/trekreview` runs a criterion\'s command only when it'],
|
||||
['agents/brief-reviewer.md', 'Anything else — the product\'s own binary (`node bin/x.mjs …`), `jq`, `grep`,'],
|
||||
['agents/brief-reviewer.md', 'Never PROPOSE such a command as a'],
|
||||
['agents/plan-critic.md', '(K10: one backticked command the criteria runner allows,'],
|
||||
['templates/trekbrief-template.md', 'One command per criterion, from the runner\'s allowlist'],
|
||||
['templates/trekbrief-template.md', 'in plain words (it runs with no shell: no quote, `$` or glob outside `node --test`).'],
|
||||
['templates/plan-template.md', 'runner allows, no pipe and no `&&` — usually `npm test` — that the executor'],
|
||||
['templates/plan-template.md', '`{a command the criteria runner allows that runs the product the way a user does'],
|
||||
['commands/trekplan.md', 'The command must also be one the criteria runner is allowed to run: a test'],
|
||||
['commands/trekplan.md', 'grep`) — with no pipe, redirect or `&&`. It runs with no shell: its words'],
|
||||
['commands/trekplan.md', 'takes a short list of read-only options, and nothing may follow'],
|
||||
];
|
||||
|
||||
test('the criteria text that moved into the skill is no longer copied into the agents, templates and /trekplan', () => {
|
||||
const back = MOVED.filter(([f, s]) => read(f).includes(s));
|
||||
assert.deepEqual(back, [], 'copied criteria text is back — it belongs to the skill');
|
||||
});
|
||||
|
||||
test('brief-reviewer and plan-critic keep their own role and point at the preloaded skill', () => {
|
||||
assert.match(read('agents/brief-reviewer.md'), /preloaded\s+`writing-criteria` skill/);
|
||||
assert.match(read('agents/brief-reviewer.md'), /BRIEF_SC_NOT_RUNNABLE/, 'flagging an unrunnable criterion stays its job');
|
||||
const d8 = section(read('agents/plan-critic.md'), /^### 8\. Verification gaps/m, /^### 9\./m);
|
||||
assert.match(d8, /preloaded\s+`writing-criteria` skill/);
|
||||
assert.match(d8, /can never go red/, 'a criterion that cannot go red is a finding');
|
||||
});
|
||||
|
||||
// --- the templates' examples run --------------------------------------------
|
||||
|
||||
// A backticked span inside a criterion line that is no placeholder (`{…}`) is
|
||||
// an example the template teaches: the runner must accept it.
|
||||
function templateExamples(text) {
|
||||
const out = [];
|
||||
for (const line of text.split('\n')) {
|
||||
if (!/^\s*-\s|\*\*(Verify|Regression|End-to-end):\*\*/.test(line)) continue;
|
||||
for (const m of line.matchAll(/`([^`]+)`/g)) {
|
||||
const span = m[1].trim();
|
||||
if (/^\{.*\}$/.test(span) || !firstCommand(`\`${span}\``).command) continue;
|
||||
out.push(span);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
test('every runnable example in the brief template\'s Success Criteria is accepted by the criteria runner', () => {
|
||||
const sc = section(read('templates/trekbrief-template.md'), /^## Success Criteria/m, /^## /m);
|
||||
const ex = templateExamples(sc);
|
||||
assert.ok(ex.length >= 3, `found ${ex.length}: ${JSON.stringify(ex)}`);
|
||||
assert.deepEqual(ex.filter((s) => !runs(s)), []);
|
||||
});
|
||||
|
||||
test('every runnable example in the plan template\'s steps and Verification is accepted by the criteria runner', () => {
|
||||
const t = read('templates/plan-template.md');
|
||||
const steps = section(t, /^## Implementation Plan/m, /^### Manifest — objective completion predicate/m);
|
||||
const ver = section(t, /^## Verification/m, /^## /m);
|
||||
const ex = [...templateExamples(steps), ...templateExamples(ver)]
|
||||
.filter((s) => !/^(git add|path\/|`)/.test(s) && !/\.(ts|mjs|js|md)$/.test(s));
|
||||
// Measured 17.10: every check in the template was a placeholder; `Regression:`
|
||||
// now shows the real command it names ("usually `npm test`").
|
||||
assert.ok(ex.includes('npm test'), `found ${ex.length}: ${JSON.stringify(ex)}`);
|
||||
assert.deepEqual(ex.filter((s) => !runs(s)), []);
|
||||
});
|
||||
|
||||
// --- the eval cases ---------------------------------------------------------
|
||||
|
||||
test('writing-criteria eval cases: 3 fire, 2 stay quiet (test-first and running the app), 1 effect case', () => {
|
||||
const dirs = readdirSync(join(ROOT, 'evals')).filter((d) => d.startsWith('skill-writing-criteria-'));
|
||||
const fires = dirs.filter((d) => /-fires-\d$/.test(d));
|
||||
const quiet = dirs.filter((d) => /-quiet-\d$/.test(d));
|
||||
const effect = dirs.filter((d) => d.endsWith('-effect'));
|
||||
assert.equal(fires.length, 3);
|
||||
assert.equal(quiet.length, 2);
|
||||
assert.equal(effect.length, 1);
|
||||
for (const d of dirs) assert.ok(existsSync(join(ROOT, 'evals', d, 'prompt.md')), `${d}/prompt.md`);
|
||||
const owners = quiet.map((d) => read(`evals/${d}/prompt.md`).match(/owner=([\w:-]+)/)?.[1]).sort();
|
||||
assert.ok(owners.includes('test-first'), 'one quiet case belongs to test-first ("write a test")');
|
||||
const quietText = quiet.map((d) => read(`evals/${d}/prompt.md`)).join('\n');
|
||||
assert.match(quietText, /\b(start|run)\b[^\n]*\bapp\b/i, 'one quiet case is "run the app" (Claude Code\'s run)');
|
||||
});
|
||||
158
tests/lib/skills-review-prose-pins.test.mjs
Normal file
158
tests/lib/skills-review-prose-pins.test.mjs
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
// tests/lib/skills-review-prose-pins.test.mjs
|
||||
// Wave 3, order 7d: the `reviewing-adversarially` skill and its wiring
|
||||
// (contract S4).
|
||||
//
|
||||
// What is pinned, and why:
|
||||
// • the skill's form is not pinned here: the form check
|
||||
// (lib/skills/skill-lint.mjs) and the routing test (lib/skills/routing.mjs)
|
||||
// run over every skill in `npm test` and are the authority (contract S1);
|
||||
// • the rules the order names stay in the skill, and its "Enforced by" says
|
||||
// which part a check holds (a finding with no source) and which part is
|
||||
// prose only (praise and flattery);
|
||||
// • the three review agents and scope-guardian preload the skill through
|
||||
// their `skills:` line, and /trekreview names it where it starts the
|
||||
// reviewers and the coordinator ("commands are the trunk");
|
||||
// • the block "Interaction Awareness — MANDATORY OVERRIDE", measured on
|
||||
// 31c77fd as the same 19 lines in brief-conformance-reviewer,
|
||||
// code-correctness-reviewer and review-coordinator, is gone from agents/:
|
||||
// the skill replaces it, and a copy coming back is drift;
|
||||
// • each agent keeps its own role (dimensions, schema, verdict).
|
||||
//
|
||||
// The behaviour itself (a separate list of what was not judged; no finding
|
||||
// without a source) was measured with helpers in the authoring session and is
|
||||
// not something a model-free test can see. The eval cases under
|
||||
// evals/skill-reviewing-adversarially-* carry it for the operator's own runs.
|
||||
// plan-critic and brief-reviewer also get the skill (S4), but their skills:
|
||||
// line belongs to order 7c and is pinned there.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { existsSync, readdirSync, readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';
|
||||
import { loadCases } from '../../lib/skills/routing.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const read = (rel) => readFileSync(join(ROOT, rel), 'utf8');
|
||||
|
||||
const SKILL = 'skills/reviewing-adversarially/SKILL.md';
|
||||
const NAME = 'voyage:reviewing-adversarially';
|
||||
const AGENTS = ['brief-conformance-reviewer', 'code-correctness-reviewer', 'review-coordinator', 'scope-guardian'];
|
||||
|
||||
function section(text, heading) {
|
||||
const parts = text.split(new RegExp(`^## ${heading}$`, 'm'));
|
||||
assert.ok(parts.length > 1, `## ${heading} not found`);
|
||||
return parts[1].split(/^## /m)[0];
|
||||
}
|
||||
|
||||
// --- the skill ---------------------------------------------------------------
|
||||
|
||||
test('reviewing-adversarially: the skill exists', () => {
|
||||
assert.ok(existsSync(join(ROOT, SKILL)), `${SKILL} missing`);
|
||||
});
|
||||
|
||||
test('reviewing-adversarially: carries the rules the order names', () => {
|
||||
const text = read(SKILL);
|
||||
for (const re of [
|
||||
/no praise/i,
|
||||
/no finding without a source/i,
|
||||
/no downgrade without a reason/i,
|
||||
/summary only/i,
|
||||
/## Declined to Judge/,
|
||||
/a claim is checked against (the )?evidence/i,
|
||||
/the diff, not the report/i,
|
||||
/axes stay separate/i,
|
||||
]) assert.match(text, re, `rule missing: ${re}`);
|
||||
});
|
||||
|
||||
test('reviewing-adversarially: "Enforced by" names the partial checks and says praise is prose only', () => {
|
||||
const enforced = section(read(SKILL), 'Enforced by');
|
||||
assert.match(enforced, /lib\/review\/findings-schema\.mjs/);
|
||||
assert.match(enforced, /lib\/review\/coordinator-contract\.mjs/);
|
||||
assert.match(enforced, /partial/i);
|
||||
assert.match(enforced, /praise[^.]*\*\*prose only\*\*/i, 'praise and flattery are prose only; say so');
|
||||
});
|
||||
|
||||
test('reviewing-adversarially: credits carry source path, commit and licence', () => {
|
||||
const credits = section(read(SKILL), 'Credits');
|
||||
for (const s of [
|
||||
'obra/superpowers @8ca22db',
|
||||
'skills/requesting-code-review/code-reviewer.md:33-48',
|
||||
'skills/verification-before-completion/SKILL.md:38-48',
|
||||
'mattpocock/skills @c55ee46',
|
||||
'skills/engineering/code-review/SKILL.md:76-78',
|
||||
]) assert.ok(credits.includes(s), `credit missing: ${s}`);
|
||||
assert.equal((credits.match(/\bMIT\b/g) || []).length, 2, 'both borrowed repositories are MIT and say so');
|
||||
});
|
||||
|
||||
// --- wiring ------------------------------------------------------------------
|
||||
|
||||
test('the three review agents and scope-guardian preload voyage:reviewing-adversarially', () => {
|
||||
for (const a of AGENTS) {
|
||||
const fm = readFrontmatterFields(read(`agents/${a}.md`)).fields;
|
||||
assert.ok(fm.skills, `agents/${a}.md has no skills: line`);
|
||||
assert.deepEqual(fm.skills.value, [NAME], `agents/${a}.md skills:`);
|
||||
// Preloading uses the skills field, not the Skill tool: the tools stay read-only.
|
||||
assert.deepEqual(fm.tools.value, ['Read', 'Glob', 'Grep'], `agents/${a}.md tools:`);
|
||||
}
|
||||
});
|
||||
|
||||
test('/trekreview names the skill where it starts the reviewers and the coordinator', () => {
|
||||
const cmd = read('commands/trekreview.md');
|
||||
const phase5 = cmd.split(/^## Phase 5 — Launch parallel reviewers$/m)[1].split(/^## Phase 6/m)[0];
|
||||
const phase6 = cmd.split(/^## Phase 6 — Coordinator dedup \+ verdict$/m)[1].split(/^## Phase 5–6 via/m)[0];
|
||||
assert.ok(phase5.includes(NAME), 'Phase 5 must name the skill');
|
||||
assert.ok(phase6.includes(NAME), 'Phase 6 must name the skill');
|
||||
});
|
||||
|
||||
test('each agent points at the preloaded skill and keeps its own role', () => {
|
||||
const own = {
|
||||
'brief-conformance-reviewer': ['## Severity rules', '## JSON output rules', 'UNIMPLEMENTED_CRITERION'],
|
||||
'code-correctness-reviewer': ['## Your 7-dimension checklist', '## JSON output rules'],
|
||||
'review-coordinator': ['## Your 4-pass process', '### Pass 4 — Compute verdict', '`## Declined to Judge` — REQUIRED'],
|
||||
'scope-guardian': ['### 2. Scope creep detection', '### Machine-readable findings block (REQUIRED)'],
|
||||
};
|
||||
for (const a of AGENTS) {
|
||||
const body = read(`agents/${a}.md`).replace(/^---\n[\s\S]*?\n---\n/, '');
|
||||
assert.match(body, /preloaded\s+`reviewing-adversarially` skill/, `agents/${a}.md must point at the preloaded skill`);
|
||||
for (const s of own[a]) assert.ok(body.includes(s), `agents/${a}.md lost its own role: ${s}`);
|
||||
}
|
||||
});
|
||||
|
||||
// --- the copied block is gone ------------------------------------------------
|
||||
|
||||
// Measured on 31c77fd: the heading, its lead sentence and the four numbered
|
||||
// rules below, the same in three agents. The skill replaces them.
|
||||
const BLOCK = [
|
||||
'Interaction Awareness — MANDATORY OVERRIDE',
|
||||
'Sycophancy is the primary vector for AI-induced harm.',
|
||||
"NEVER reformulate a user's statement in stronger terms than they used.",
|
||||
'**NEVER start a response with** "Absolutely", "Exactly", "Great point",',
|
||||
'**Before endorsing any plan:** identify at least one real risk or weakness.',
|
||||
'**When the user asks "right?" or "don\'t you think?":** evaluate independently.',
|
||||
];
|
||||
|
||||
test('the Interaction Awareness block and its four rules are in no agent file', () => {
|
||||
const back = [];
|
||||
for (const f of readdirSync(join(ROOT, 'agents')).filter((x) => x.endsWith('.md'))) {
|
||||
const t = read(`agents/${f}`);
|
||||
for (const s of BLOCK) if (t.includes(s)) back.push(`${f}: ${s}`);
|
||||
}
|
||||
assert.deepEqual(back, [], 'the copied block is back in an agent; it belongs to the skill');
|
||||
});
|
||||
|
||||
// --- the eval cases ----------------------------------------------------------
|
||||
|
||||
test('reviewing-adversarially eval cases: the negatives cover a plain code review and a proofread', () => {
|
||||
const mine = loadCases(ROOT).filter((c) => c.skill === 'reviewing-adversarially');
|
||||
const n = (k) => mine.filter((c) => c.kind === k);
|
||||
assert.ok(n('trigger').length >= 3, `need >= 3 should-trigger cases, have ${n('trigger').length}`);
|
||||
assert.ok(n('no-trigger').length >= 2, `need >= 2 should-not-trigger cases, have ${n('no-trigger').length}`);
|
||||
assert.equal(n('effect').length, 1);
|
||||
const quiet = n('no-trigger').map((c) => c.prompt).join('\n');
|
||||
assert.match(quiet, /\/code-review/, 'one negative is an ordinary /code-review request');
|
||||
assert.match(quiet, /wording|proofread|typos/i, 'one negative is a "look over my text" request');
|
||||
for (const c of mine) assert.doesNotMatch(read(`evals/${c.case}/prompt.md`), /\bproevesett\b/, 'skill cases are not part of the fixed test set');
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue