fix: the trial's mechanical faults -- approval label, import path, stats, annotate path, red kind
voyage-lekerepo end-to-end trial (2026-10-06), report points 8, 10, 12, 15, 16:
- approval-log: «Approve (Recommended)» / «Godkjenn (anbefalt)» is the
approval it labels; the suffix never turns a hollow answer into a yes.
- trekexecute Phase 8: the state-file block imports atomic-write through
${CLAUDE_PLUGIN_ROOT}; a relative import only ran with cwd = plugin root
(the test runs the block from another directory).
- trekreview, trekresearch, trekcontinue: stats go through stats-append like
the other commands, and a failed write is reported, never skipped
silently (closes the open STATE finding for all three). The schema
fixture's stale line refs for these and trekplan are re-measured.
- annotate prints an absolute path, so file://{$ANNOT_HTML} is a live link.
- red-proof: `red_kind` = load-error | test-failure on a proven red.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
609eb35015
commit
81dc5af680
9 changed files with 191 additions and 24 deletions
8
tests/fixtures/jsonl-schemas.md
vendored
8
tests/fixtures/jsonl-schemas.md
vendored
|
|
@ -20,13 +20,13 @@
|
|||
| schema_id | fields | writer_path | line_ref | v4.1 additive | PII |
|
||||
|-----------|--------|-------------|----------|---------------|-----|
|
||||
| trekbrief-stats | ts, task, slug, mode, interview_turns, review_iterations, brief_quality, research_topics, auto_research, auto_result, intent_approved, project_dir, order_id, order_claimed_at, order_reason | commands/trekbrief.md (orchestrator-emit Phase 7; order_* added by lib/stats/stats-append.mjs) | trekbrief.md:1124-1139 | profile, phase_models, profile_source | none |
|
||||
| trekresearch-stats | ts, question, mode, scope, engine, slug, project_dir, brief_path, dimensions, dimensions_baseline, dimensions_baseline_preserved, effort, conv_turns, empty_turns, unique_sources, agents_local, agents_external, gemini_used, confidence, contradictions, open_questions | commands/trekresearch.md (orchestrator-emit Stats tracking) | trekresearch.md:634-676 | profile, phase_models, parallel_agents, external_research_enabled, profile_source | none |
|
||||
| trekplan-stats | ts, task, mode, slug, brief_path, project_dir, codebase_size, codebase_files, agents_deployed, deep_dives, research_briefs_used, research_scout_used, critic_verdict, guardian_verdict, outcome | commands/trekplan.md (orchestrator-emit Phase 12) | trekplan.md:805-826 | profile, phase_models, parallel_agents, profile_source | none |
|
||||
| trekresearch-stats | ts, question, mode, scope, engine, slug, project_dir, brief_path, dimensions, dimensions_baseline, dimensions_baseline_preserved, effort, conv_turns, empty_turns, unique_sources, agents_local, agents_external, gemini_used, confidence, contradictions, open_questions | lib/stats/stats-append.mjs (trekresearch.md Stats tracking) | trekresearch.md:782-808 | profile, phase_models, parallel_agents, external_research_enabled, profile_source | none |
|
||||
| trekplan-stats | ts, task, mode, slug, brief_path, project_dir, codebase_size, codebase_files, agents_deployed, deep_dives, research_briefs_used, research_scout_used, critic_verdict, guardian_verdict, outcome | lib/stats/stats-append.mjs (trekplan.md Phase 12) | trekplan.md:1115-1135 | profile, phase_models, parallel_agents, profile_source | none |
|
||||
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2330-2352 | profile, phase_models, profile_source | none |
|
||||
| trekexecute-stats (autonomy events) | ts, event, known_event, payload | lib/stats/event-emit.mjs `emit()` | event-emit.mjs:78-99 | payload.profile, payload.phase_models, payload.profile_source | none |
|
||||
| trekexecute-stats (PostToolUse Bash) | ts, session_id, command_excerpt, duration_ms, success | hooks/scripts/post-bash-stats.mjs (Bash PostToolUse) | post-bash-stats.mjs:42-54 | none (hook is plugin-level, not profile-aware) | command_excerpt (CWE-212) |
|
||||
| trekreview-stats | ts, slug, verdict, counts (BLOCKER/MAJOR/MINOR/SUGGESTION), reviewed_files_count, mode, duration_ms | commands/trekreview.md (orchestrator-emit Phase 8) | trekreview.md:255 | profile, phase_models, profile_source | none |
|
||||
| trekcontinue-stats | ts, project, next_session_label, status | commands/trekcontinue.md (orchestrator-emit Phase 5) | trekcontinue.md:289 | profile, profile_source | none |
|
||||
| trekreview-stats | ts, slug, verdict, counts (BLOCKER/MAJOR/MINOR/SUGGESTION), reviewed_files_count, mode, duration_ms | lib/stats/stats-append.mjs (trekreview.md Phase 8) | trekreview.md:628-632 | profile, phase_models, profile_source | none |
|
||||
| trekcontinue-stats | ts, project, next_session_label, status | lib/stats/stats-append.mjs (trekcontinue.md Phase 5) | trekcontinue.md:293-297 | profile, profile_source | none |
|
||||
| token-usage | ts, session_id, scope, model, tokens_input, tokens_output, tokens_cache_creation, tokens_cache_read, cost_usd, is_estimate, price_table_version | lib/stats/token-usage.mjs `captureTokenUsage` (via hooks/scripts/otel-export.mjs Stop hook, opt-in `VOYAGE_TOKEN_METER`) | token-usage.mjs:buildRecord | n/a (SKAL-2 schema) | session_id (stripped at export) |
|
||||
|
||||
## Field-allowlist input for Step 11
|
||||
|
|
|
|||
130
tests/lib/trial-mechanical-fixes.test.mjs
Normal file
130
tests/lib/trial-mechanical-fixes.test.mjs
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
// tests/lib/trial-mechanical-fixes.test.mjs
|
||||
// Mechanical faults from the voyage-lekerepo end-to-end trial (2026-10-06),
|
||||
// report points 8, 10, 12, 15 and 16. Each test fails on the code the trial
|
||||
// ran.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, existsSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join, isAbsolute } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { isExplicitApproval } from '../../lib/util/approval-log.mjs';
|
||||
import { proveRed } from '../../lib/verification/red-proof.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const read = (p) => readFileSync(join(ROOT, p), 'utf8');
|
||||
|
||||
// --- 8. the label AskUserQuestion recommends is still an approval -----------
|
||||
|
||||
test('approval: «Approve (Recommended)» is the explicit approval it labels', () => {
|
||||
for (const a of ['Approve (Recommended)', 'Approve (recommended)', 'Godkjenn (anbefalt)', 'Approve (Recommended).']) {
|
||||
assert.equal(isExplicitApproval(a).approved, true, a);
|
||||
}
|
||||
});
|
||||
|
||||
test('approval: the suffix does not turn a hollow answer into a yes', () => {
|
||||
for (const a of ['Sounds good (Recommended)', 'Høres bra ut (anbefalt)', '(Recommended)', 'Revise (Recommended)']) {
|
||||
assert.equal(isExplicitApproval(a).approved, false, a);
|
||||
}
|
||||
});
|
||||
|
||||
// --- 10. the state-file block runs from any working directory ---------------
|
||||
|
||||
test('trekexecute Phase 8: the state-file block imports through the plugin root and runs outside it', () => {
|
||||
const md = read('commands/trekexecute.md');
|
||||
assert.doesNotMatch(md, /from '\.\/lib\//, 'a relative import only works with cwd = plugin root');
|
||||
const start = md.indexOf('node --input-type=module -e "\nimport path from \'node:path\';');
|
||||
assert.ok(start > 0, 'the state-file block');
|
||||
const end = md.indexOf('\n```', start);
|
||||
const block = md.slice(start, end)
|
||||
.replaceAll('${CLAUDE_PLUGIN_ROOT}', ROOT)
|
||||
.replace("'{project_dir}' '{next_session_brief_path}' '{next_session_label}' '{status}'", '"$1" brief.md "Session 2" ready');
|
||||
const elsewhere = mkdtempSync(join(tmpdir(), 'voyage-state-'));
|
||||
try {
|
||||
const r = spawnSync('/bin/bash', ['-c', block, 'x', elsewhere], { cwd: elsewhere, encoding: 'utf8' });
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
assert.ok(existsSync(join(elsewhere, '.session-state.local.json')));
|
||||
assert.match(readFileSync(join(elsewhere, 'NEXT-SESSION-PROMPT.local.md'), 'utf8'), /produced_by: trekexecute/);
|
||||
} finally {
|
||||
rmSync(elsewhere, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// --- 12. every command's stats go through the one writer --------------------
|
||||
|
||||
test('stats: trekreview, trekresearch and trekcontinue write through stats-append, never silently skipped', () => {
|
||||
for (const [file, kind] of [['commands/trekreview.md', 'trekreview'], ['commands/trekresearch.md', 'trekresearch'], ['commands/trekcontinue.md', 'trekcontinue']]) {
|
||||
const md = read(file);
|
||||
assert.match(md, new RegExp(`node \\$\\{CLAUDE_PLUGIN_ROOT\\}/lib/stats/stats-append\\.mjs ${kind} <<'JSON'`), file);
|
||||
assert.doesNotMatch(md, /skip (stats )?silently|silently skip/i, file);
|
||||
assert.doesNotMatch(md, new RegExp(`\\$\\{CLAUDE_PLUGIN_DATA\\}/${kind}-stats\\.jsonl`), file);
|
||||
}
|
||||
});
|
||||
|
||||
// --- 15. annotate prints a path a file:// link can use ----------------------
|
||||
|
||||
test('annotate: a relative input still prints an absolute .html path', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'voyage-annot-'));
|
||||
try {
|
||||
writeFileSync(join(dir, 'brief.md'), '# Brief\n\nText.\n');
|
||||
const r = spawnSync(process.execPath, [join(ROOT, 'scripts', 'annotate.mjs'), 'brief.md'], { cwd: dir, encoding: 'utf8' });
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
const out = r.stdout.trim();
|
||||
assert.ok(isAbsolute(out), out);
|
||||
assert.ok(existsSync(out), out);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// --- 16. the proof says what kind of red it was ------------------------------
|
||||
|
||||
function sh(cwd, cmd, args) {
|
||||
const r = spawnSync(cmd, args, { cwd, encoding: 'utf8' });
|
||||
assert.equal(r.status, 0, `${cmd} ${args.join(' ')}: ${r.stderr}`);
|
||||
return r.stdout.trim();
|
||||
}
|
||||
|
||||
function repo(moduleText) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'red-kind-'));
|
||||
sh(dir, 'git', ['init', '-q']);
|
||||
sh(dir, 'git', ['config', 'user.email', 't@example.invalid']);
|
||||
sh(dir, 'git', ['config', 'user.name', 'test']);
|
||||
mkdirSync(join(dir, 'lib'));
|
||||
writeFileSync(join(dir, 'lib', 'sum.mjs'), moduleText);
|
||||
sh(dir, 'git', ['add', 'lib/sum.mjs']);
|
||||
sh(dir, 'git', ['commit', '-q', '-m', 'base']);
|
||||
mkdirSync(join(dir, 'tests'));
|
||||
writeFileSync(join(dir, 'tests', 'sum.test.mjs'), [
|
||||
"import { test } from 'node:test';",
|
||||
"import { strict as assert } from 'node:assert';",
|
||||
"import { sum } from '../lib/sum.mjs';",
|
||||
"test('sum adds', () => { assert.equal(sum(1, 2), 3); });",
|
||||
'',
|
||||
].join('\n'));
|
||||
writeFileSync(join(dir, 'lib', 'sum.mjs'), 'export function sum(a, b) { return a + b; }\n');
|
||||
return { dir, base: sh(dir, 'git', ['rev-parse', 'HEAD']) };
|
||||
}
|
||||
|
||||
const CMD = 'node --test tests/sum.test.mjs';
|
||||
|
||||
test('red proof: a start point that cannot even load the test is proven with red_kind "load-error"', () => {
|
||||
const { dir, base } = repo('export function other() { return 0; }\n');
|
||||
try {
|
||||
const r = proveRed({ cwd: dir, base, cmd: CMD, tests: ['tests/sum.test.mjs'] });
|
||||
assert.equal(r.status, 'proven', JSON.stringify(r));
|
||||
assert.equal(r.red_kind, 'load-error');
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('red proof: a named test that fails at the start point is red_kind "test-failure"', () => {
|
||||
const { dir, base } = repo('export function sum(a, b) { return a - b; }\n');
|
||||
try {
|
||||
const r = proveRed({ cwd: dir, base, cmd: CMD, tests: ['tests/sum.test.mjs'] });
|
||||
assert.equal(r.status, 'proven', JSON.stringify(r));
|
||||
assert.equal(r.red_kind, 'test-failure');
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue