fix: the fix round before 6.3.0 (order 10)
1. A step of only tests can never be proven red. One reading, stepTestsOnly (lib/parsers/plan-schema.mjs, test paths from the new lib/util/test-path.mjs): the plan validator warns PLAN_STEP_TESTS_ONLY, red-proof --plan/--step adds tests_only to a not-red-at-base, and /trekexecute and the step helper send such a step straight up to the plan instead of into fix rounds that cannot succeed. /trekplan, the plan template and plan-critic (dimension 8, still 12) say the test belongs in the step that builds the behaviour. 2. Timing tests under load: the hook helper's kill limit is 30 s and per call overridable; SC #14 passes on the fastest of up to three runs against the same 800 ms limit (tests/helpers/timing.mjs). 3. The example plan's On failure lines open with a K11 word. 4. /trekreview saves the reviewer's whole answer for the findings check and names FINDING_PROSE_LINE_OUT_OF_RANGE (error: re-ask; warning: report). 5. The criteria runner's commands run without NODE_TEST_CONTEXT (commandEnv, moved to lib/util/command-env.mjs). 6. The resolver returns effort only when the brief's phase signal sets one; the four commands said it always did. Also: the K2 hook tests used fixed session ids in a per-user temp dir, so two suites running at once took each other's approval records; the ids are now unique per run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
19ff772773
commit
33b58f460a
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
25 changed files with 680 additions and 74 deletions
2
tests/fixtures/jsonl-schemas.md
vendored
2
tests/fixtures/jsonl-schemas.md
vendored
|
|
@ -22,7 +22,7 @@
|
|||
| trekbrief-stats | ts, task, slug, mode, interview_turns, review_iterations, brief_quality, research_topics, auto_research, auto_result, intent_approved, project_dir, order_id, order_claimed_at, order_reason | commands/trekbrief.md (orchestrator-emit Phase 7; order_* added by lib/stats/stats-append.mjs) | trekbrief.md:1130-1145 | profile, phase_models, profile_source | none |
|
||||
| trekresearch-stats | ts, question, mode, scope, engine, slug, project_dir, brief_path, dimensions, dimensions_baseline, dimensions_baseline_preserved, effort, conv_turns, empty_turns, unique_sources, agents_local, agents_external, gemini_used, confidence, contradictions, open_questions | lib/stats/stats-append.mjs (trekresearch.md Stats tracking) | trekresearch.md:782-808 | profile, phase_models, parallel_agents, external_research_enabled, profile_source | none |
|
||||
| trekplan-stats | ts, task, mode, slug, brief_path, project_dir, codebase_size, codebase_files, agents_deployed, deep_dives, research_briefs_used, research_scout_used, critic_verdict, guardian_verdict, outcome | lib/stats/stats-append.mjs (trekplan.md Phase 12) | trekplan.md:1115-1135 | profile, phase_models, parallel_agents, profile_source | none |
|
||||
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2855-2877 | profile, phase_models, profile_source | none |
|
||||
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2856-2878 | profile, phase_models, profile_source | none |
|
||||
| trekexecute-stats (autonomy events) | ts, event, known_event, payload | lib/stats/event-emit.mjs `emit()` | event-emit.mjs:78-99 | payload.profile, payload.phase_models, payload.profile_source | none |
|
||||
| trekexecute-stats (PostToolUse Bash) | ts, session_id, command_excerpt, duration_ms, success | hooks/scripts/post-bash-stats.mjs (Bash PostToolUse) | post-bash-stats.mjs:42-54 | none (hook is plugin-level, not profile-aware) | command_excerpt (CWE-212) |
|
||||
| trekreview-stats | ts, slug, verdict, counts (BLOCKER/MAJOR/MINOR/SUGGESTION), reviewed_files_count, mode, duration_ms | lib/stats/stats-append.mjs (trekreview.md Phase 8) | trekreview.md:628-632 | profile, phase_models, profile_source | none |
|
||||
|
|
|
|||
|
|
@ -1,11 +1,20 @@
|
|||
// hook-helper.mjs — Shared test helper for hook scripts.
|
||||
// Spawns a hook as a child process and feeds it JSON via stdin.
|
||||
//
|
||||
// Source: ../../../llm-security/tests/hooks/hook-helper.mjs (verbatim copy)
|
||||
// Source: ../../../llm-security/tests/hooks/hook-helper.mjs (copied; since order 10
|
||||
// the kill limit is HOOK_TIMEOUT_MS and a call can set its own)
|
||||
// Provenance: borrowed within the same marketplace (same author, MIT).
|
||||
|
||||
import { execFile } from 'node:child_process';
|
||||
|
||||
// How long a hook may run before the helper kills it. The tests check a
|
||||
// hook's answer, not its speed (SC #14 times itself, tests/helpers/timing.mjs),
|
||||
// so the limit only has to catch a hook that hangs. 5 s killed P7
|
||||
// (post-edit-check) at load 95-177 in four wave sessions while it was 8/8
|
||||
// green alone (order 10, 2026-10-09). hooks/hooks.json sets no timeout for
|
||||
// these hooks, so 5 s was stricter than anything the hook meets in use.
|
||||
export const HOOK_TIMEOUT_MS = 30_000;
|
||||
|
||||
/**
|
||||
* Run a hook script by spawning `node <scriptPath>` and piping `input` to stdin.
|
||||
*
|
||||
|
|
@ -23,15 +32,16 @@ export function runHook(scriptPath, input) {
|
|||
* @param {string} scriptPath - Absolute path to the hook .mjs file
|
||||
* @param {object|string} input - JSON payload (object will be stringified)
|
||||
* @param {Record<string, string>} envOverrides - Extra env vars to set
|
||||
* @param {{ timeoutMs?: number }} [opts] - kill limit for this call (default HOOK_TIMEOUT_MS)
|
||||
* @returns {Promise<{ code: number, stdout: string, stderr: string }>}
|
||||
*/
|
||||
export function runHookWithEnv(scriptPath, input, envOverrides) {
|
||||
export function runHookWithEnv(scriptPath, input, envOverrides, opts = {}) {
|
||||
return new Promise((resolve) => {
|
||||
const env = { ...process.env, ...envOverrides };
|
||||
const child = execFile(
|
||||
'node',
|
||||
[scriptPath],
|
||||
{ timeout: 5000, env },
|
||||
{ timeout: opts.timeoutMs ?? HOOK_TIMEOUT_MS, env },
|
||||
(err, stdout, stderr) => {
|
||||
resolve({
|
||||
code: child.exitCode ?? (err && err.code === 'ERR_CHILD_PROCESS_STDIO_FINAL' ? 0 : 1),
|
||||
|
|
|
|||
25
tests/helpers/timing.mjs
Normal file
25
tests/helpers/timing.mjs
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
// tests/helpers/timing.mjs
|
||||
// A timing assertion that tolerates a busy machine without loosening its limit.
|
||||
//
|
||||
// Load only ever ADDS time to a run, so the fastest of a few runs is the best
|
||||
// reading of what the code itself costs. `fastestWithin` measures up to
|
||||
// `tries` runs and passes as soon as one is under the limit. A real slowness
|
||||
// is slow every time and still fails; one run slowed by four other test suites
|
||||
// does not (measured S123 and in wave 2, 2026-10-08: SC #14 red under load
|
||||
// 95-177, green alone).
|
||||
|
||||
/**
|
||||
* @param {number} limitMs - the limit; a run must be strictly under it
|
||||
* @param {() => Promise<number>} measure - runs once, resolves to its elapsed ms
|
||||
* @param {number} [tries=3]
|
||||
* @returns {Promise<{ok: boolean, fastest: number, runs: number[]}>}
|
||||
*/
|
||||
export async function fastestWithin(limitMs, measure, tries = 3) {
|
||||
const runs = [];
|
||||
for (let i = 0; i < tries; i++) {
|
||||
runs.push(await measure());
|
||||
if (runs[runs.length - 1] < limitMs) break;
|
||||
}
|
||||
const fastest = Math.min(...runs);
|
||||
return { ok: fastest < limitMs, fastest, runs };
|
||||
}
|
||||
|
|
@ -27,12 +27,16 @@ gitIn(REPO, 'checkout', '-q', '-b', 'feat');
|
|||
const sha = (s) => createHash('sha256').update(s).digest('hex');
|
||||
|
||||
let callNo = 0;
|
||||
// Approval records live in one temp dir per user (hooks/lib/approval-record.mjs),
|
||||
// keyed by session_id + tool_use_id, so two suites running at once (two
|
||||
// worktrees) took each other's records with fixed ids. Unique per run.
|
||||
const RUN = `${process.pid}-${Date.now()}`;
|
||||
|
||||
// A Bash call goes through the same two hooks as in a session: the guard asks
|
||||
// (and records what it asked), the call runs, then the approval is logged.
|
||||
async function post(root, input, env = {}) {
|
||||
const logFile = join(tmp(), 'stub.jsonl');
|
||||
const ids = { session_id: 'sess-log', tool_use_id: `toolu_log_${(callNo += 1)}` };
|
||||
const ids = { session_id: `sess-log-${RUN}`, tool_use_id: `toolu_log_${(callNo += 1)}` };
|
||||
if (input.tool_name === 'Bash') {
|
||||
await runHookWithEnv(join(root, 'hooks', 'scripts', 'pre-bash-executor.mjs'),
|
||||
{ hook_event_name: 'PreToolUse', cwd: REPO, ...ids, tool_name: 'Bash', tool_input: input.tool_input }, env);
|
||||
|
|
|
|||
|
|
@ -16,6 +16,9 @@ const PRE_WRITE = join(ROOT, 'hooks', 'scripts', 'pre-write-executor.mjs');
|
|||
const STOP = join(ROOT, 'hooks', 'scripts', 'stop-test-lock.mjs');
|
||||
const LOCK = join(ROOT, 'lib', 'util', 'test-lock.mjs');
|
||||
const STEP = join(ROOT, 'lib', 'verification', 'step-check.mjs');
|
||||
// Approval records share one temp dir per user, keyed by session_id +
|
||||
// tool_use_id: fixed ids let two suites running at once take each other's.
|
||||
const RUN = `${process.pid}-${Date.now()}`;
|
||||
const { classifyGitApproval } = await import(join(ROOT, 'hooks', 'lib', 'git-approval.mjs'));
|
||||
|
||||
const FEAT = makeRepo('main');
|
||||
|
|
@ -81,7 +84,7 @@ test('review 8: the log records what the person was asked, not a re-reading afte
|
|||
const repo = makeRepo('main');
|
||||
gitIn(repo, 'checkout', '-q', '-b', 'feat');
|
||||
const cmd = 'git checkout main && git merge feat';
|
||||
const base = { session_id: 's-8', tool_use_id: 'toolu_8', cwd: repo, tool_name: 'Bash', tool_input: { command: cmd } };
|
||||
const base = { session_id: `s-8-${RUN}`, tool_use_id: 'toolu_8', cwd: repo, tool_name: 'Bash', tool_input: { command: cmd } };
|
||||
const pre = await runHook(join(plugin, 'hooks', 'scripts', 'pre-bash-executor.mjs'), { ...base, hook_event_name: 'PreToolUse' });
|
||||
assert.match(pre.stdout, /merge into main/);
|
||||
gitIn(repo, 'checkout', '-q', 'main'); // the command ran
|
||||
|
|
@ -97,7 +100,7 @@ test('review 8: a command nobody was asked about is not logged, even if it reads
|
|||
const repo = makeRepo('main');
|
||||
const log = join(tmp(), 'stub.jsonl');
|
||||
await runHookWithEnv(join(plugin, 'hooks', 'scripts', 'post-approval-log.mjs'), {
|
||||
hook_event_name: 'PostToolUse', session_id: 's-9', tool_use_id: 'toolu_never_asked', cwd: repo,
|
||||
hook_event_name: 'PostToolUse', session_id: `s-9-${RUN}`, tool_use_id: 'toolu_never_asked', cwd: repo,
|
||||
tool_name: 'Bash', tool_input: { command: 'git push origin main' },
|
||||
}, { STUB_LOG: log });
|
||||
assert.equal(readStubLog(log).length, 0);
|
||||
|
|
@ -110,7 +113,7 @@ test('review 8: a command nobody was asked about is not logged, even if it reads
|
|||
test('uncertain (no-prompt mode): the push to main is refused, and no approval is logged', async () => {
|
||||
const plugin = pluginCopy({ 'lib/util/approval-log.mjs': 'stub' });
|
||||
const repo = makeRepo('main');
|
||||
const base = { session_id: 's-b', tool_use_id: 'toolu_b', cwd: repo, tool_name: 'Bash', tool_input: { command: 'git push origin main' }, permission_mode: 'bypassPermissions' };
|
||||
const base = { session_id: `s-b-${RUN}`, tool_use_id: 'toolu_b', cwd: repo, tool_name: 'Bash', tool_input: { command: 'git push origin main' }, permission_mode: 'bypassPermissions' };
|
||||
const pre = await runHook(join(plugin, 'hooks', 'scripts', 'pre-bash-executor.mjs'), { ...base, hook_event_name: 'PreToolUse' });
|
||||
assert.equal(pre.code, 2, `refused, not asked: ${JSON.stringify(pre)}`);
|
||||
const log = join(tmp(), 'stub.jsonl');
|
||||
|
|
@ -297,7 +300,7 @@ test('review 2nd round (N4): when the record cannot be written, the prompt says
|
|||
const fileAsDir = join(tmp(), 'not-a-dir');
|
||||
writeFileSync(fileAsDir, 'x');
|
||||
const r = await runHookWithEnv(join(plugin, 'hooks', 'scripts', 'pre-bash-executor.mjs'), {
|
||||
hook_event_name: 'PreToolUse', session_id: 's-n4', tool_use_id: 'toolu_n4', cwd: makeRepo('main'),
|
||||
hook_event_name: 'PreToolUse', session_id: `s-n4-${RUN}`, tool_use_id: 'toolu_n4', cwd: makeRepo('main'),
|
||||
tool_name: 'Bash', tool_input: { command: 'git push origin main' },
|
||||
}, { TMPDIR: fileAsDir });
|
||||
const reason = parseJsonOut(r.stdout).hookSpecificOutput.permissionDecisionReason;
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { join, dirname } from 'node:path';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { runHookWithEnv } from '../helpers/hook-helper.mjs';
|
||||
import { fastestWithin } from '../helpers/timing.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const HOOK_PATH = join(__dirname, '..', '..', 'hooks', 'scripts', 'otel-export.mjs');
|
||||
|
|
@ -104,16 +105,21 @@ test('SC #14: VOYAGE_EXPORT_MODE=otlp + invalid endpoint → stderr [voyage] war
|
|||
test('SC #14: tail-latency for textfile mode < 200ms (NFR)', async () => {
|
||||
const dataDir = setupDataDir();
|
||||
try {
|
||||
const start = performance.now();
|
||||
const r = await runHookWithEnv(HOOK_PATH, '{}', {
|
||||
VOYAGE_EXPORT_MODE: 'textfile',
|
||||
CLAUDE_PLUGIN_DATA: dataDir,
|
||||
// The fastest of up to three cold runs against the same 800 ms limit
|
||||
// (tests/helpers/timing.mjs): load only adds time, so a real slowness
|
||||
// still fails every run, and one run slowed by a busy machine does not.
|
||||
const t = await fastestWithin(800, async () => {
|
||||
const start = performance.now();
|
||||
const r = await runHookWithEnv(HOOK_PATH, '{}', {
|
||||
VOYAGE_EXPORT_MODE: 'textfile',
|
||||
CLAUDE_PLUGIN_DATA: dataDir,
|
||||
});
|
||||
assert.equal(r.code, 0);
|
||||
return performance.now() - start;
|
||||
});
|
||||
const elapsed = performance.now() - start;
|
||||
assert.equal(r.code, 0);
|
||||
// 200ms NFR with extra headroom for cold-start node spawn (~100ms typical)
|
||||
assert.ok(elapsed < 800,
|
||||
`textfile export tail-latency too slow: ${elapsed.toFixed(0)}ms (NFR <200ms in-process; <800ms allowed for cold spawn)`);
|
||||
assert.ok(t.ok,
|
||||
`textfile export tail-latency too slow: fastest ${t.fastest.toFixed(0)}ms of ${t.runs.map((x) => x.toFixed(0)).join('/')}ms (NFR <200ms in-process; <800ms allowed for cold spawn)`);
|
||||
} finally {
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
|
|
|
|||
131
tests/lib/retterunde-prose-pins.test.mjs
Normal file
131
tests/lib/retterunde-prose-pins.test.mjs
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
// tests/lib/retterunde-prose-pins.test.mjs
|
||||
// Prose pins for the fix round before 6.3.0 (order 10) whose fix is text a
|
||||
// session reads. Counted as doc pins by lib/util/test-census.mjs (the file name).
|
||||
// 1b A step of only tests can never be proven red: the plan text, the
|
||||
// template and plan-critic say the test belongs in the step that builds
|
||||
// the behaviour (voyage-lekerepo trial 3, 2026-10-09, finding 1).
|
||||
// 1c The executor and the step helper pass `--plan`/`--step` to red-proof
|
||||
// and send a `tests_only` `not-red-at-base` up to the plan, not into fix
|
||||
// rounds that cannot succeed.
|
||||
// 4 /trekreview names FINDING_PROSE_LINE_OUT_OF_RANGE and hands the check
|
||||
// the reviewer's whole answer, not only its JSON block.
|
||||
// 6 The resolver returns `effort` only when the brief's phase signal sets
|
||||
// one; no command says it always does.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { findSection } from '../../lib/parsers/plan-schema.mjs';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const read = (p) => readFileSync(join(ROOT, p), 'utf8');
|
||||
|
||||
/** The text of a heading's section (level from its #-count), up to the next one outside a fence. */
|
||||
function section(text, heading) {
|
||||
const level = heading.match(/^#+/)[0].length;
|
||||
const s = findSection(text, heading.replace(/^#+ /, ''), level);
|
||||
assert.ok(s, `missing ${heading}`);
|
||||
return s.text;
|
||||
}
|
||||
|
||||
const RULE = /test\s+for\s+a\s+behaviou?r\s+belongs\s+in\s+the\s+step\s+that\s+builds\s+it/i;
|
||||
|
||||
// --- 1b --------------------------------------------------------------------------
|
||||
|
||||
test('1b: /trekplan says the test belongs in the step that builds the behaviour, and names the warning', () => {
|
||||
const plan = read('commands/trekplan.md');
|
||||
assert.match(plan, RULE);
|
||||
// At the vertical-slice rule and at the step field list.
|
||||
assert.match(plan, /Steps are vertical slices[\s\S]{0,900}only tests[\s\S]{0,300}never be proven red/i);
|
||||
assert.match(plan, /\*\*Test first:\*\* — [^\n]*\n[^\n]*step that builds/i);
|
||||
assert.match(plan, /PLAN_STEP_TESTS_ONLY/);
|
||||
});
|
||||
|
||||
test('1b: the plan template carries the same rule', () => {
|
||||
const t = read('templates/plan-template.md');
|
||||
assert.match(t, RULE);
|
||||
assert.match(t, /only tests[\s\S]{0,200}never be proven red/i);
|
||||
});
|
||||
|
||||
test('1b: plan-critic has it as a finding under Verification gaps, and still 12 dimensions', () => {
|
||||
const critic = read('agents/plan-critic.md');
|
||||
const d8 = section(critic, '### 8. Verification gaps');
|
||||
assert.match(d8, /PLAN_STEP_TESTS_ONLY/);
|
||||
assert.match(d8, /only tests[\s\S]{0,400}\*\*major\*\*/i);
|
||||
assert.equal((critic.match(/^### \d+\. /gm) ?? []).length, 12);
|
||||
});
|
||||
|
||||
// --- 1c --------------------------------------------------------------------------
|
||||
|
||||
test('1c: the executor runs red-proof with --plan/--step and sends a tests_only not-red-at-base up to the plan', () => {
|
||||
const ex = read('commands/trekexecute.md');
|
||||
assert.match(ex, /red-proof\.mjs --base \{start_sha\} --cmd "\{Verify command\}" --tests [^\n]*--plan "\{plan_path\}" --step \{N\}[^\n]*--json/);
|
||||
assert.match(ex, /`not-red-at-base`, `tests_only: true`[^\n]*straight up to the plan \(Sub-step F, `fix_rounds = 4`, stop class 4\)/);
|
||||
// The helper's gate, as the executor describes it, carries the same flags.
|
||||
assert.match(ex, /red-proof\.mjs --red-only --base \{start_sha\} [^\n]*--plan "\{plan_path\}" --step \{N\}/);
|
||||
// The other not-red-at-base keeps its fix round.
|
||||
assert.match(ex, /`not-red-at-base`, otherwise[^\n]*finding for Sub-step F/);
|
||||
});
|
||||
|
||||
test('1c: the step helper passes --plan/--step to its red gate and does not rewrite a tests_only test', () => {
|
||||
const h = read('agents/step-implementer.md');
|
||||
assert.match(h, /red-proof\.mjs --red-only [^\n]*--plan "\{plan\}" --step \{step\}/);
|
||||
assert.match(h, /`tests_only: true`[\s\S]{0,300}do not rewrite/i);
|
||||
});
|
||||
|
||||
// --- 4 ---------------------------------------------------------------------------
|
||||
|
||||
test('4: /trekreview names FINDING_PROSE_LINE_OUT_OF_RANGE: re-ask on the error, report the warning', () => {
|
||||
const r = read('commands/trekreview.md');
|
||||
assert.match(r, /FINDING_PROSE_LINE_OUT_OF_RANGE/);
|
||||
assert.match(r, /FINDING_PROSE_LINE_OUT_OF_RANGE[\s\S]{0,600}re-ask/i);
|
||||
assert.match(r, /FINDING_PROSE_LINE_OUT_OF_RANGE[\s\S]{0,900}warning[\s\S]{0,300}report/i);
|
||||
});
|
||||
|
||||
test('4: /trekreview saves the reviewer\'s whole answer for the check, not only the JSON block', () => {
|
||||
const r = read('commands/trekreview.md');
|
||||
assert.match(r, /\{project_dir\}\/\.review-out-\{reviewer\}\.md/);
|
||||
assert.match(r, /whole answer[\s\S]{0,300}not only the\s+(JSON\s+)?block/i);
|
||||
assert.match(r, /findings-schema\.mjs --json --repo-root [^\n]*"\{project_dir\}\/\.review-out-\{reviewer\}\.md"/);
|
||||
assert.doesNotMatch(r, /Collect each reviewer's trailing JSON block/);
|
||||
});
|
||||
|
||||
test('4: what the check is given decides: the whole answer warns on a prose citation, the block alone cannot', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'retterunde-review-'));
|
||||
try {
|
||||
mkdirSync(join(dir, 'src'));
|
||||
writeFileSync(join(dir, 'src', 'a.mjs'), 'export const a = 1;\n');
|
||||
const block = '```json\n{"reviewer":"code-correctness-reviewer","findings":[]}\n```\n';
|
||||
writeFileSync(join(dir, 'whole.md'), `The bug is at \`src/a.mjs:40\`.\n\n${block}`);
|
||||
writeFileSync(join(dir, 'block.md'), block);
|
||||
const cli = (f) => spawnSync(process.execPath,
|
||||
[join(ROOT, 'lib', 'review', 'findings-schema.mjs'), '--json', '--repo-root', dir, join(dir, f)], { encoding: 'utf8' });
|
||||
const whole = JSON.parse(cli('whole.md').stdout);
|
||||
assert.deepEqual(whole.warnings.map((w) => w.code), ['FINDING_PROSE_LINE_OUT_OF_RANGE']);
|
||||
assert.deepEqual(JSON.parse(cli('block.md').stdout).warnings, []);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
// --- 6 ---------------------------------------------------------------------------
|
||||
|
||||
const RESOLVER_COMMANDS = ['trekplan', 'trekexecute', 'trekresearch', 'trekreview'];
|
||||
|
||||
test('6: no command says the resolver always returns {effort, model, source}', () => {
|
||||
for (const f of readdirSync(join(ROOT, 'commands')).filter((x) => x.endsWith('.md'))) {
|
||||
const t = read(`commands/${f}`);
|
||||
assert.doesNotMatch(t, /\{effort, model, source\}/, f);
|
||||
}
|
||||
});
|
||||
|
||||
test('6: each resolver-using command says `effort` comes only from the brief signal, else standard', () => {
|
||||
for (const c of RESOLVER_COMMANDS) {
|
||||
const t = read(`commands/${c}.md`);
|
||||
const hits = t.match(/\{model, source\}[^\n]*\n?[^\n]*effort[^\n]*only when the brief's phase signal sets one/g) ?? [];
|
||||
assert.ok(hits.length >= 1, `${c}: no statement that effort is only present with a brief signal`);
|
||||
assert.match(t, /without it[^\n]*\n?[^\n]*`?standard`?/i, c);
|
||||
}
|
||||
});
|
||||
198
tests/lib/retterunde.test.mjs
Normal file
198
tests/lib/retterunde.test.mjs
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
// tests/lib/retterunde.test.mjs
|
||||
// The fix round before 6.3.0 (order 10), the points that are behaviour, not
|
||||
// prose. Each test was red before its fix.
|
||||
// 1 A step whose `Files:` are all tests can never be proven red: the
|
||||
// behaviour it tests is already at its start point (voyage-lekerepo
|
||||
// trial 3, 2026-10-09, finding 1). (a) the validator warns
|
||||
// PLAN_STEP_TESTS_ONLY; (c) red-proof says `tests_only` on its
|
||||
// `not-red-at-base`, so the executor sends the step up to the plan instead
|
||||
// of into fix rounds that cannot succeed. One reading: stepTestsOnly.
|
||||
// 3 The example plan's `On failure:` lines open with a K11 word.
|
||||
// 5 The criteria runner's commands never see NODE_TEST_CONTEXT: a nested
|
||||
// `node --test` would report to the parent and exit 0 on a red test.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { validatePlan, validatePlanContent } from '../../lib/validators/plan-validator.mjs';
|
||||
import { stepTestsOnly } from '../../lib/parsers/plan-schema.mjs';
|
||||
import { proveRed } from '../../lib/verification/red-proof.mjs';
|
||||
import { runCriteria } from '../../lib/verification/criteria-runner.mjs';
|
||||
import { buildPlan } from '../helpers/plan-v18-builder.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const codes = (list) => list.map((e) => e.code);
|
||||
|
||||
// --- 1 (a): the validator -----------------------------------------------------
|
||||
|
||||
// Step 2 changes only tests for behaviour step 1 delivered, as in the trial.
|
||||
const TESTS_ONLY_PLAN = buildPlan({
|
||||
steps: [{}, { file: 'tests/e2e.sh', test: 'tests/e2e-guard.test.mjs', endToEnd: 'bash tests/e2e.sh' }],
|
||||
});
|
||||
|
||||
test('1a: a step whose Files are all tests gives PLAN_STEP_TESTS_ONLY, a warning on that step', () => {
|
||||
const r = validatePlanContent(TESTS_ONLY_PLAN);
|
||||
const w = r.warnings.filter((x) => x.code === 'PLAN_STEP_TESTS_ONLY');
|
||||
assert.equal(w.length, 1, codes(r.warnings).join(','));
|
||||
assert.match(w[0].message, /Step 2/);
|
||||
assert.ok(!codes(r.errors).includes('PLAN_STEP_TESTS_ONLY'), 'a warning, never an error');
|
||||
assert.equal(r.valid, true, codes(r.errors).join(','));
|
||||
});
|
||||
|
||||
test('1a: a step with code and its test gives no PLAN_STEP_TESTS_ONLY (as before)', () => {
|
||||
const r = validatePlanContent(buildPlan());
|
||||
assert.ok(!codes(r.warnings).includes('PLAN_STEP_TESTS_ONLY'), codes(r.warnings).join(','));
|
||||
});
|
||||
|
||||
test('1a: the reading - Files all tests (by path or as the Test first file) or not; no Files = no warning', () => {
|
||||
const body = (files, tf = 'tests/x.test.mjs') => [
|
||||
'### Step 1: x', '',
|
||||
...(files === null ? [] : [`- **Files:** ${files}`]),
|
||||
'- **Test first:**', ` - File: \`${tf}\` (new)`, ' - Verifies: x',
|
||||
].join('\n');
|
||||
assert.equal(stepTestsOnly(body('`tests/e2e.sh`, `tests/e2e-guard.sh`')).testsOnly, true);
|
||||
assert.equal(stepTestsOnly(body('`src/a.test.ts` (new file), `lib/b.spec.js`, `pkg/c_test.go`')).testsOnly, true);
|
||||
assert.equal(stepTestsOnly(body('`__tests__/a.js`, `test/b.js`')).testsOnly, true);
|
||||
// The Test first file counts as a test wherever it lives.
|
||||
assert.equal(stepTestsOnly(body('`scripts/check.sh`', 'scripts/check.sh')).testsOnly, true);
|
||||
assert.equal(stepTestsOnly(body('`src/cli.mjs`, `tests/cli.test.mjs`')).testsOnly, false);
|
||||
assert.equal(stepTestsOnly(body('`src/cli.mjs`')).testsOnly, false);
|
||||
assert.equal(stepTestsOnly(body(null)).testsOnly, false);
|
||||
assert.deepEqual(stepTestsOnly(body('`tests/e2e.sh`, `tests/e2e-guard.sh`')).files, ['tests/e2e.sh', 'tests/e2e-guard.sh']);
|
||||
});
|
||||
|
||||
test('1a: a plain path beside a backticked one is still read - a step that also changes code is not tests-only', () => {
|
||||
const body = (files) => ['### Step 1: x', '', `- **Files:** ${files}`].join('\n');
|
||||
const mixed = stepTestsOnly(body('src/cli.mjs (modify), `tests/x.test.mjs` (new)'));
|
||||
assert.deepEqual(mixed.files, ['src/cli.mjs', 'tests/x.test.mjs']);
|
||||
assert.equal(mixed.testsOnly, false);
|
||||
const list = stepTestsOnly(['### Step 1: x', '', '- **Files:**', ' - `tests/x.test.mjs` (new)', ' - src/cli.mjs'].join('\n'));
|
||||
assert.equal(list.testsOnly, false, JSON.stringify(list.files));
|
||||
});
|
||||
|
||||
test('1a: a document under spec/ or specs/ is not a test for this reading', () => {
|
||||
const body = (files) => ['### Step 1: x', '', `- **Files:** ${files}`].join('\n');
|
||||
assert.equal(stepTestsOnly(body('`docs/specs/api.md`, `specs/001-auth/plan.md`')).testsOnly, false);
|
||||
assert.equal(stepTestsOnly(body('`spec/cli_spec.rb`')).testsOnly, true);
|
||||
});
|
||||
|
||||
// --- 1 (c): red-proof says tests_only on not-red-at-base ----------------------
|
||||
|
||||
function sh(cwd, cmd, args) {
|
||||
const r = spawnSync(cmd, args, { cwd, encoding: 'utf8' });
|
||||
assert.equal(r.status, 0, `${cmd} ${args.join(' ')}: ${r.stderr}`);
|
||||
return r.stdout.trim();
|
||||
}
|
||||
|
||||
const SUM_TEST = [
|
||||
"import { test } from 'node:test';",
|
||||
"import { strict as assert } from 'node:assert';",
|
||||
"import { sum } from '../lib/sum.mjs';",
|
||||
"test('sum adds', () => { assert.equal(sum(1, 2), 3); });",
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
/** The behaviour is already at the start point; the step adds only its test. */
|
||||
function greenAtBase() {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'retterunde-red-'));
|
||||
sh(dir, 'git', ['init', '-q']);
|
||||
sh(dir, 'git', ['config', 'user.email', 't@example.invalid']);
|
||||
sh(dir, 'git', ['config', 'user.name', 'test']);
|
||||
mkdirSync(join(dir, 'lib'));
|
||||
writeFileSync(join(dir, 'lib', 'sum.mjs'), 'export function sum(a, b) { return a + b; }\n');
|
||||
sh(dir, 'git', ['add', 'lib/sum.mjs']);
|
||||
sh(dir, 'git', ['commit', '-q', '-m', 'base']);
|
||||
mkdirSync(join(dir, 'tests'));
|
||||
writeFileSync(join(dir, 'tests', 'sum.test.mjs'), SUM_TEST);
|
||||
return { dir, base: sh(dir, 'git', ['rev-parse', 'HEAD']) };
|
||||
}
|
||||
|
||||
function planWithStep2Files(dir, files) {
|
||||
const p = join(dir, 'plan.local.md');
|
||||
writeFileSync(p, buildPlan({ steps: [{}, { file: files, test: 'tests/sum.test.mjs', endToEnd: 'bash tests/e2e.sh' }] }));
|
||||
return p;
|
||||
}
|
||||
|
||||
const CMD = 'node --test tests/sum.test.mjs';
|
||||
|
||||
test('1c: not-red-at-base on a step whose Files are all tests says tests_only: true (red-only and full)', () => {
|
||||
const { dir, base } = greenAtBase();
|
||||
try {
|
||||
const plan = planWithStep2Files(dir, 'tests/sum.test.mjs');
|
||||
for (const redOnly of [true, false]) {
|
||||
const r = proveRed({ cwd: dir, base, cmd: CMD, tests: ['tests/sum.test.mjs'], redOnly, plan, step: 2 });
|
||||
assert.equal(r.reason, 'not-red-at-base', JSON.stringify(r));
|
||||
assert.equal(r.tests_only, true, JSON.stringify(r));
|
||||
}
|
||||
const cli = spawnSync(process.execPath, [join(ROOT, 'lib', 'verification', 'red-proof.mjs'),
|
||||
'--cwd', dir, '--base', base, '--cmd', CMD, '--tests', 'tests/sum.test.mjs', '--plan', plan, '--step', '2', '--json'],
|
||||
{ encoding: 'utf8' });
|
||||
assert.equal(cli.status, 1, cli.stdout + cli.stderr);
|
||||
assert.equal(JSON.parse(cli.stdout).tests_only, true);
|
||||
// Text mode names the field too (the step helper reads it there).
|
||||
const text = spawnSync(process.execPath, [join(ROOT, 'lib', 'verification', 'red-proof.mjs'),
|
||||
'--red-only', '--cwd', dir, '--base', base, '--cmd', CMD, '--tests', 'tests/sum.test.mjs', '--plan', plan, '--step', '2'],
|
||||
{ encoding: 'utf8' });
|
||||
assert.match(text.stdout, /tests_only: true/);
|
||||
// --step without --plan is a usage error, never silently dropped.
|
||||
const lone = spawnSync(process.execPath, [join(ROOT, 'lib', 'verification', 'red-proof.mjs'),
|
||||
'--cwd', dir, '--base', base, '--cmd', CMD, '--tests', 'tests/sum.test.mjs', '--step', '2'], { encoding: 'utf8' });
|
||||
assert.equal(lone.status, 2, lone.stderr);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('1c: not-red-at-base on a step that also changes code says tests_only: false (fix rounds, as before)', () => {
|
||||
const { dir, base } = greenAtBase();
|
||||
try {
|
||||
const plan = planWithStep2Files(dir, 'lib/sum.mjs');
|
||||
const r = proveRed({ cwd: dir, base, cmd: CMD, tests: ['tests/sum.test.mjs'], plan, step: 2 });
|
||||
assert.equal(r.reason, 'not-red-at-base');
|
||||
assert.equal(r.tests_only, false, JSON.stringify(r));
|
||||
// Without --plan nothing new is said: the field is absent.
|
||||
const bare = proveRed({ cwd: dir, base, cmd: CMD, tests: ['tests/sum.test.mjs'] });
|
||||
assert.equal(bare.reason, 'not-red-at-base');
|
||||
assert.equal('tests_only' in bare, false);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
// --- 3: the example plan reads with the K11 words -----------------------------
|
||||
|
||||
test('3: the example plan gives no PLAN_STEP_ON_FAILURE_UNKNOWN', () => {
|
||||
const r = validatePlan(join(ROOT, 'examples', '01-add-verbose-flag', 'plan.md'));
|
||||
assert.deepEqual(r.warnings.filter((w) => w.code === 'PLAN_STEP_ON_FAILURE_UNKNOWN').map((w) => w.message), []);
|
||||
});
|
||||
|
||||
// --- 5: the criteria runner strips NODE_TEST_CONTEXT ---------------------------
|
||||
|
||||
test('5: a nested `node --test` on a failing test is red even when NODE_TEST_CONTEXT is set', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'retterunde-ctx-'));
|
||||
const saved = process.env.NODE_TEST_CONTEXT;
|
||||
try {
|
||||
mkdirSync(join(dir, 'tests'));
|
||||
writeFileSync(join(dir, 'tests', 'fail.test.mjs'), [
|
||||
"import { test } from 'node:test';",
|
||||
"import { strict as assert } from 'node:assert';",
|
||||
"test('fails', () => { assert.equal(1, 2); });",
|
||||
'',
|
||||
].join('\n'));
|
||||
process.env.NODE_TEST_CONTEXT = 'child-v8';
|
||||
const [r] = runCriteria([{ label: 'C1', text: 'x', command: 'node --test tests/fail.test.mjs' }], { cwd: dir });
|
||||
assert.equal(r.status, 'failed', JSON.stringify(r));
|
||||
assert.notEqual(r.exitCode, 0);
|
||||
} finally {
|
||||
if (saved === undefined) delete process.env.NODE_TEST_CONTEXT;
|
||||
else process.env.NODE_TEST_CONTEXT = saved;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// Guard against a reading of the example that counts nothing: it has steps.
|
||||
test('3: the example plan is read (six or more steps)', () => {
|
||||
const text = readFileSync(join(ROOT, 'examples', '01-add-verbose-flag', 'plan.md'), 'utf8');
|
||||
assert.ok(validatePlanContent(text).parsed.steps.length >= 6);
|
||||
});
|
||||
63
tests/lib/timing-under-load.test.mjs
Normal file
63
tests/lib/timing-under-load.test.mjs
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
// tests/lib/timing-under-load.test.mjs
|
||||
// The two timing tests that went red under load in the wave sessions (order 10,
|
||||
// point 2; operator 2026-10-09): P7 (post-edit-check) was killed by the hook
|
||||
// helper's 5 s limit at load 95-177 and was 8/8 green alone; SC #14
|
||||
// (otel-export) demands < 800 ms for ONE cold run.
|
||||
// - The helper's kill limit is longer (HOOK_TIMEOUT_MS) and overridable per
|
||||
// call. P7 checks the hook's answer, not its speed, so that weakens nothing.
|
||||
// - SC #14 keeps its 800 ms limit and measures up to three runs, passing on
|
||||
// the fastest. Load only ever ADDS time, so the fastest run is the best
|
||||
// reading of the real cost: a real slowness is slow every time and still
|
||||
// fails; one run slowed by a busy machine does not.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { fastestWithin } from '../helpers/timing.mjs';
|
||||
import { HOOK_TIMEOUT_MS, runHookWithEnv } from '../helpers/hook-helper.mjs';
|
||||
|
||||
/** A measure that returns the given durations in turn (no real waiting). */
|
||||
const stub = (durations) => {
|
||||
let i = 0;
|
||||
return async () => durations[i++];
|
||||
};
|
||||
|
||||
test('timing: a stub that is slow every time fails the limit (a real slowness is felled)', async () => {
|
||||
const r = await fastestWithin(800, stub([900, 950, 1200]));
|
||||
assert.equal(r.ok, false);
|
||||
assert.deepEqual(r.runs, [900, 950, 1200]);
|
||||
assert.equal(r.fastest, 900);
|
||||
});
|
||||
|
||||
test('timing: one slow run of three is tolerated - the next run under the limit passes', async () => {
|
||||
const r = await fastestWithin(800, stub([1500, 300, 2000]));
|
||||
assert.equal(r.ok, true);
|
||||
assert.deepEqual(r.runs, [1500, 300], 'stops at the first run under the limit');
|
||||
});
|
||||
|
||||
test('timing: the limit itself is not loosened - a run AT the limit is not under it', async () => {
|
||||
const r = await fastestWithin(800, stub([800, 800, 800]));
|
||||
assert.equal(r.ok, false);
|
||||
});
|
||||
|
||||
test('timing: a first run under the limit passes at once (one run on a quiet machine)', async () => {
|
||||
const r = await fastestWithin(800, stub([120]));
|
||||
assert.equal(r.ok, true);
|
||||
assert.deepEqual(r.runs, [120]);
|
||||
});
|
||||
|
||||
test('hook helper: the kill limit is far above 5 s by default, and a call can set its own', async () => {
|
||||
assert.ok(HOOK_TIMEOUT_MS >= 30_000, `HOOK_TIMEOUT_MS = ${HOOK_TIMEOUT_MS}`);
|
||||
const dir = mkdtempSync(join(tmpdir(), 'hook-helper-'));
|
||||
try {
|
||||
const slow = join(dir, 'slow-hook.mjs');
|
||||
writeFileSync(slow, "setTimeout(() => { process.stdout.write('done'); process.exit(0); }, 400);\n");
|
||||
const killed = await runHookWithEnv(slow, '{}', {}, { timeoutMs: 100 });
|
||||
assert.notEqual(killed.code, 0, 'a per-call limit below the hook\'s run time kills it');
|
||||
const done = await runHookWithEnv(slow, '{}', {});
|
||||
assert.equal(done.code, 0);
|
||||
assert.equal(done.stdout, 'done');
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue