fix: the second lekerepo trial's faults that made a result wrong or misleading
voyage-lekerepo trial 2 (2026-10-08), report section A; the operator decided
they are fixed before the next release:
- A1: a /code-review finding whose category is simplification, efficiency,
style or readability enters as CODE_REVIEW_IMPROVEMENT (MINOR), not MAJOR.
- A2: recommended_action is the engine's own fix when transcribed, otherwise
a test-first step; never the failure scenario dressed as a remedy.
- A3: a /code-review finding outside the exact range it was given is declined
and listed, but no longer holds ALLOW back; /security-review's still does.
- A4: the manifest must_contain recipe uses grep -E -e ... -- so a pattern
like --sep is not read as an option.
- A5: indented sub-points belong to their criterion (counted), and a
criterion with sub-points and no command warns BRIEF_SC_SUBPOINTS.
- A6: a brief requiring bash 3.2 whose criterion runs plain bash warns
BRIEF_SC_BASH_NOT_PINNED; /trekbrief proposes /bin/bash.
- A7: the plan template's and /trekplan's checkpoints stage the step's files.
Tests fail on 5f9faee first (tests/lib/trial2-fixes.test.mjs, 11 of 14).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
8094b00861
commit
7c11c2aa9f
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
18 changed files with 300 additions and 39 deletions
4
tests/fixtures/jsonl-schemas.md
vendored
4
tests/fixtures/jsonl-schemas.md
vendored
|
|
@ -19,10 +19,10 @@
|
|||
|
||||
| schema_id | fields | writer_path | line_ref | v4.1 additive | PII |
|
||||
|-----------|--------|-------------|----------|---------------|-----|
|
||||
| trekbrief-stats | ts, task, slug, mode, interview_turns, review_iterations, brief_quality, research_topics, auto_research, auto_result, intent_approved, project_dir, order_id, order_claimed_at, order_reason | commands/trekbrief.md (orchestrator-emit Phase 7; order_* added by lib/stats/stats-append.mjs) | trekbrief.md:1124-1139 | profile, phase_models, profile_source | none |
|
||||
| trekbrief-stats | ts, task, slug, mode, interview_turns, review_iterations, brief_quality, research_topics, auto_research, auto_result, intent_approved, project_dir, order_id, order_claimed_at, order_reason | commands/trekbrief.md (orchestrator-emit Phase 7; order_* added by lib/stats/stats-append.mjs) | trekbrief.md:1130-1145 | profile, phase_models, profile_source | none |
|
||||
| trekresearch-stats | ts, question, mode, scope, engine, slug, project_dir, brief_path, dimensions, dimensions_baseline, dimensions_baseline_preserved, effort, conv_turns, empty_turns, unique_sources, agents_local, agents_external, gemini_used, confidence, contradictions, open_questions | lib/stats/stats-append.mjs (trekresearch.md Stats tracking) | trekresearch.md:782-808 | profile, phase_models, parallel_agents, external_research_enabled, profile_source | none |
|
||||
| trekplan-stats | ts, task, mode, slug, brief_path, project_dir, codebase_size, codebase_files, agents_deployed, deep_dives, research_briefs_used, research_scout_used, critic_verdict, guardian_verdict, outcome | lib/stats/stats-append.mjs (trekplan.md Phase 12) | trekplan.md:1115-1135 | profile, phase_models, parallel_agents, profile_source | none |
|
||||
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2343-2365 | profile, phase_models, profile_source | none |
|
||||
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2344-2366 | profile, phase_models, profile_source | none |
|
||||
| trekexecute-stats (autonomy events) | ts, event, known_event, payload | lib/stats/event-emit.mjs `emit()` | event-emit.mjs:78-99 | payload.profile, payload.phase_models, payload.profile_source | none |
|
||||
| trekexecute-stats (PostToolUse Bash) | ts, session_id, command_excerpt, duration_ms, success | hooks/scripts/post-bash-stats.mjs (Bash PostToolUse) | post-bash-stats.mjs:42-54 | none (hook is plugin-level, not profile-aware) | command_excerpt (CWE-212) |
|
||||
| trekreview-stats | ts, slug, verdict, counts (BLOCKER/MAJOR/MINOR/SUGGESTION), reviewed_files_count, mode, duration_ms | lib/stats/stats-append.mjs (trekreview.md Phase 8) | trekreview.md:628-632 | profile, phase_models, profile_source | none |
|
||||
|
|
|
|||
|
|
@ -269,12 +269,12 @@ test('HANDOVER-CONTRACTS.md Handover 7 § Lifecycle names --cleanup and produced
|
|||
// file" (commands/trekcontinue.md exists, so that loop already asserts the
|
||||
// /trekcontinue reference). Single-command existence pin → redundant.
|
||||
|
||||
test('rule-catalogue has exactly 14 entries', async () => {
|
||||
test('rule-catalogue has exactly 15 entries', async () => {
|
||||
const mod = await import('../../lib/review/rule-catalogue.mjs');
|
||||
assert.strictEqual(
|
||||
mod.RULE_CATALOGUE.length,
|
||||
14,
|
||||
'lib/review/rule-catalogue.mjs RULE_CATALOGUE size invariant: must be 14 (12 v1.0 baseline + 2 engine keys, ordre 5)',
|
||||
15,
|
||||
'lib/review/rule-catalogue.mjs RULE_CATALOGUE size invariant: must be 15 (12 v1.0 baseline + 2 engine keys, ordre 5, + CODE_REVIEW_IMPROVEMENT, lekerepo trial 2)',
|
||||
);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,9 @@ test('engine finding: recommended_action is an imperative built from the failure
|
|||
const f = r.payload.findings[0];
|
||||
assert.doesNotMatch(f.recommended_action, /^Resolve:/);
|
||||
assert.notEqual(f.recommended_action.replace(/^\w+\s*/, ''), f.title);
|
||||
assert.match(f.recommended_action, /^Fix /);
|
||||
// Trial 2 (2026-10-08): the action is a test-first step, not the scenario
|
||||
// dressed as a remedy (tests/lib/trial2-fixes.test.mjs, A2).
|
||||
assert.match(f.recommended_action, /^Write a test /);
|
||||
assert.match(f.recommended_action, /bin\/slug\.mjs:12/);
|
||||
assert.match(f.recommended_action, /prints `x---max`/);
|
||||
});
|
||||
|
|
@ -44,7 +46,7 @@ test('engine finding: recommended_action is an imperative built from the failure
|
|||
test('engine finding without a failure scenario still gets an action that is not the title', () => {
|
||||
const r = toReviewerPayload({ engine: 'security-review', findings: [{ file: 'a.mjs', line: 0, summary: 'token logged' }] });
|
||||
const f = r.payload.findings[0];
|
||||
assert.match(f.recommended_action, /^Fix /);
|
||||
assert.match(f.recommended_action, /^Write a test /);
|
||||
assert.notEqual(f.recommended_action, `Resolve: ${f.title}`);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -39,13 +39,14 @@ test('RULE_CATALOGUE — all category values within enum', () => {
|
|||
}
|
||||
});
|
||||
|
||||
test('RULE_KEYS.size === RULE_CATALOGUE.length (== 14) — pinned by doc-consistency', () => {
|
||||
test('RULE_KEYS.size === RULE_CATALOGUE.length (== 15) — pinned by doc-consistency', () => {
|
||||
assert.equal(RULE_KEYS.size, RULE_CATALOGUE.length);
|
||||
assert.equal(RULE_CATALOGUE.length, 14);
|
||||
assert.equal(RULE_CATALOGUE.length, 15);
|
||||
});
|
||||
|
||||
test('RULE_CATALOGUE — the two Claude Code engine keys exist at their tiers (ordre 5)', () => {
|
||||
assert.equal(getRule('CODE_REVIEW_DEFECT')?.severity, 'MAJOR');
|
||||
assert.equal(getRule('CODE_REVIEW_IMPROVEMENT')?.severity, 'MINOR');
|
||||
assert.equal(getRule('SECURITY_REVIEW_VULNERABILITY')?.severity, 'BLOCKER');
|
||||
});
|
||||
|
||||
|
|
|
|||
172
tests/lib/trial2-fixes.test.mjs
Normal file
172
tests/lib/trial2-fixes.test.mjs
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
// tests/lib/trial2-fixes.test.mjs
|
||||
// The voyage-lekerepo second end-to-end trial (2026-10-08), report section A:
|
||||
// faults that make a result wrong or misleading. The operator decided they are
|
||||
// fixed before the next release. Each test fails on main 5f9faee.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { toReviewerPayload } from '../../lib/review/engine-findings.mjs';
|
||||
import { runContract } from '../../lib/review/coordinator-contract.mjs';
|
||||
import { getRule } from '../../lib/review/rule-catalogue.mjs';
|
||||
import { parseSuccessCriteria } from '../../lib/verification/criteria-runner.mjs';
|
||||
import { validateBriefContent } from '../../lib/validators/brief-validator.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const read = (p) => readFileSync(join(ROOT, p), 'utf8');
|
||||
const codes = (list) => list.map((e) => e.code);
|
||||
|
||||
const engineFinding = (extra = {}) => ({
|
||||
file: 'src/slug.mjs', line: 9,
|
||||
summary: 'A two-step detour where one split/filter/join would do',
|
||||
failure_scenario: 'slug("a--b") walks the string twice',
|
||||
...extra,
|
||||
});
|
||||
|
||||
// --- A1. an improvement from /code-review is not a MAJOR defect --------------
|
||||
|
||||
test('A1: a /code-review simplification or efficiency finding is MINOR, a correctness finding stays MAJOR', () => {
|
||||
const r = toReviewerPayload({ engine: 'code-review', findings: [
|
||||
engineFinding({ category: 'simplification' }),
|
||||
engineFinding({ line: 10, category: 'efficiency' }),
|
||||
engineFinding({ line: 11, category: 'correctness' }),
|
||||
engineFinding({ line: 12 }),
|
||||
] });
|
||||
assert.equal(r.ok, true, r.errors.join('; '));
|
||||
const sev = r.payload.findings.map((f) => f.severity);
|
||||
assert.deepEqual(sev, ['MINOR', 'MINOR', 'MAJOR', 'MAJOR']);
|
||||
for (const f of r.payload.findings) assert.equal(f.severity, getRule(f.rule_key).severity, 'severity is fixed by rule_key');
|
||||
});
|
||||
|
||||
test('A1: /security-review keeps its tier whatever the category', () => {
|
||||
const r = toReviewerPayload({ engine: 'security-review', findings: [engineFinding({ category: 'simplification' })] });
|
||||
assert.equal(r.payload.findings[0].severity, 'BLOCKER');
|
||||
});
|
||||
|
||||
// --- A2. recommended_action is a remedy, not the failure scenario -----------
|
||||
|
||||
test('A2: the engine\'s own fix text becomes the recommended action', () => {
|
||||
const r = toReviewerPayload({ engine: 'code-review', findings: [engineFinding({ fix: 'Use one split/filter/join pass' })] });
|
||||
assert.equal(r.payload.findings[0].recommended_action, 'Use one split/filter/join pass');
|
||||
});
|
||||
|
||||
test('A2: without a fix text the action is a step: reproduce the scenario in a test, then change the location', () => {
|
||||
const r = toReviewerPayload({ engine: 'code-review', findings: [engineFinding()] });
|
||||
const a = r.payload.findings[0].recommended_action;
|
||||
assert.doesNotMatch(a, /so this no longer happens/);
|
||||
assert.match(a, /^Write a test that reproduces/);
|
||||
assert.match(a, /src\/slug\.mjs:9/);
|
||||
});
|
||||
|
||||
test('A2: /trekreview asks the transcription for the engine\'s fix text and forbids inventing one', () => {
|
||||
const md = read('commands/trekreview.md');
|
||||
const p = md.slice(md.indexOf('## Phase 4.7'), md.indexOf('## Phase 5'));
|
||||
assert.match(p, /the\s+fix\s+the\s+engine\s+proposes\s+\(`fix`\)/);
|
||||
assert.match(p, /never\s+write\s+one\s+the\s+engine\s+did\s+not\s+give/);
|
||||
});
|
||||
|
||||
// --- A3. an out-of-range /code-review finding does not lock the verdict -----
|
||||
|
||||
test('A3: a /code-review finding outside the exact range it was given is declined, and does not hold ALLOW back', () => {
|
||||
const out = { ...engineFinding({ file: 'tests/cli.test.mjs', line: 1 }), engine: 'code-review', rule_key: 'CODE_REVIEW_DEFECT', severity: 'MAJOR', title: 'No CLI test for --sep' };
|
||||
const c = runContract([], { engineOutOfScope: [out] });
|
||||
assert.equal(c.verdict, 'ALLOW', JSON.stringify(c.allow_blocked_by));
|
||||
assert.equal(c.declined_to_judge.length, 1);
|
||||
assert.equal(c.declined_to_judge[0].reason, 'engine:out-of-scope');
|
||||
});
|
||||
|
||||
test('A3: a /security-review finding outside the range still forbids ALLOW (its base is wider and not ours)', () => {
|
||||
const out = { ...engineFinding({ file: 'lib/auth.mjs' }), engine: 'security-review', rule_key: 'SECURITY_REVIEW_VULNERABILITY', severity: 'BLOCKER', title: 'Token logged' };
|
||||
const c = runContract([], { engineOutOfScope: [out] });
|
||||
assert.notEqual(c.verdict, 'ALLOW');
|
||||
});
|
||||
|
||||
// --- A4. the manifest's must_contain check survives a pattern that starts with "-"
|
||||
|
||||
test('A4: the must_contain recipe passes the pattern with -e and ends options with --', () => {
|
||||
const md = read('commands/trekexecute.md');
|
||||
assert.match(md, /`grep -E -e "\{pattern\}" -- \{path\}`/);
|
||||
assert.doesNotMatch(md, /`grep -E "\{pattern\}" \{path\}`/);
|
||||
const dir = mkdtempSync(join(tmpdir(), 'voyage-grep-'));
|
||||
try {
|
||||
writeFileSync(join(dir, 'README.md'), 'Use --sep to choose the separator.\n');
|
||||
const r = spawnSync('/bin/bash', ['-c', 'grep -E -e "--sep" -- README.md'], { cwd: dir, encoding: 'utf8' });
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
// --- A5. sub-points belong to their criterion -------------------------------
|
||||
|
||||
function brief(sections) {
|
||||
return [
|
||||
'---', 'type: trekbrief', 'brief_version: "2.2"', 'task: t', 'slug: s',
|
||||
'research_topics: 0', 'research_status: skipped', 'framing: new-direction', 'phase_signals_partial: true', '---', '',
|
||||
'## TL;DR', '', 'A toy.', '', '## Intent', '', 'Why.', '', '## Goal', '', 'What.', '',
|
||||
...sections,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
const NESTED = [
|
||||
'## Success Criteria', '',
|
||||
'- `npm test` exits 0.',
|
||||
'- The separator is honoured:',
|
||||
' - `-` joins words',
|
||||
' - `_` joins words',
|
||||
' - `.` joins words',
|
||||
'- `bash tests/e2e.sh` exits 0.', '',
|
||||
];
|
||||
|
||||
test('A5: indented sub-points are part of their criterion, not criteria of their own', () => {
|
||||
const sc = parseSuccessCriteria(brief(NESTED));
|
||||
assert.deepEqual(sc.map((c) => c.label), ['SC1', 'SC2', 'SC3']);
|
||||
assert.equal(sc[1].subpoints, 3);
|
||||
assert.equal(sc[2].command, 'bash tests/e2e.sh');
|
||||
});
|
||||
|
||||
test('A5: a criterion with sub-points and no command of its own is a warning that names it', () => {
|
||||
const r = validateBriefContent(brief(NESTED));
|
||||
const w = r.warnings.filter((x) => x.code === 'BRIEF_SC_SUBPOINTS');
|
||||
assert.equal(w.length, 1, JSON.stringify(r.warnings));
|
||||
assert.match(w[0].message, /SC2/);
|
||||
});
|
||||
|
||||
// --- A6. a bash 3.2 constraint is measured under bash 3.2 --------------------
|
||||
|
||||
const BASH32 = ['## Constraints', '', '- Scripts must run under bash 3.2 (macOS system bash).', ''];
|
||||
|
||||
test('A6: a brief that requires bash 3.2 and runs a criterion with plain `bash` gets a warning pointing at /bin/bash', () => {
|
||||
const r = validateBriefContent(brief([...BASH32, '## Success Criteria', '', '- `bash tests/contract.sh` exits 0.', '']));
|
||||
const w = r.warnings.filter((x) => x.code === 'BRIEF_SC_BASH_NOT_PINNED');
|
||||
assert.equal(w.length, 1, JSON.stringify(r.warnings));
|
||||
assert.match(w[0].hint, /\/bin\/bash tests\//);
|
||||
});
|
||||
|
||||
test('A6: `/bin/bash`, or no bash 3.2 constraint, raises no such warning', () => {
|
||||
const pinned = validateBriefContent(brief([...BASH32, '## Success Criteria', '', '- `/bin/bash tests/contract.sh` exits 0.', '']));
|
||||
assert.ok(!codes(pinned.warnings).includes('BRIEF_SC_BASH_NOT_PINNED'));
|
||||
const free = validateBriefContent(brief(['## Success Criteria', '', '- `bash tests/contract.sh` exits 0.', '']));
|
||||
assert.ok(!codes(free.warnings).includes('BRIEF_SC_BASH_NOT_PINNED'));
|
||||
});
|
||||
|
||||
test('A6: /trekbrief proposes /bin/bash when the brief requires bash 3.2', () => {
|
||||
const md = read('commands/trekbrief.md');
|
||||
assert.match(md, /requires\s+bash\s+3\.2[^`]*`\/bin\/bash tests\/<script>\.sh`/);
|
||||
assert.match(md, /BRIEF_SC_BASH_NOT_PINNED/);
|
||||
assert.match(md, /BRIEF_SC_SUBPOINTS/);
|
||||
});
|
||||
|
||||
// --- A7. the plan template's checkpoint stages what it commits --------------
|
||||
|
||||
test('A7: every checkpoint the plan template and /trekplan\'s example show stages the step\'s files first', () => {
|
||||
const lines = [
|
||||
...read('templates/plan-template.md').split('\n'),
|
||||
...read('commands/trekplan.md').split('\n'),
|
||||
].filter((l) => /\*\*Checkpoint:\*\*/.test(l) && /`git [^`]*commit -m /.test(l));
|
||||
assert.ok(lines.length >= 3, `found ${lines.length}`);
|
||||
for (const l of lines) assert.match(l, /`git add [^`]+ && git commit -m /, l);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue