fix: round 6 of the release review of 6.3.0
Round 6 (v6.2.0..c3641ff, WARN: 11 MAJOR, 3 MINOR; 0 BLOCKER). The round-5
fixes hold (code-correctness-reviewer); three of the new MAJOR were side
effects of running checks with no shell. Each fixed behind a test that
failed first on c3641ff.
- git checks stay in the repo too: no path that climbs out with .., starts
at / or reads <rev>:../x, and no --no-index (both reviewers).
- A trailing `# comment` on a shell-fence line is dropped: with no shell,
its words made an honest check NOT RUN, among them the first check of the
shipped example plan.
- A glob reaches `node --test` only on Node 21 and later, which expands it
itself; on an older Node it is refused with that reason (engines >=18).
- With no shell, exit 126/127 is the test program's own: red proof reads it
as a red (the control run still tells an environment fault apart), the
done line and the test lock as a failure.
- A runner that could not start (not installed) is NOT RUN in the criteria
runner and not-run (exit 2) in regression-check, as red-proof and
done-line already said; never a red suite.
- Tests-only: a backticked `Makefile` alone in its entry is a path, and red
proof confirms the plan's tests-only reading against the step's own diff
(the altitude finding: the diff is a fact, the Files: prose is not).
- release-check stops a suite that never ends (15 minutes) and reads it as
not measured (exit 2).
- The two lines this release added to pre-bash-executor.mjs are ASCII.
- MINOR: git and the other runners share one plain-word pattern.
Not in this round: MINOR f00a2eb8 (shared norm/ticked-path and isMain
helpers) and MINOR on plan-review-dedup's repeated tokenizing.
Suite 2194 (2192/0/2).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
c3641ffa42
commit
45aa9e107a
17 changed files with 262 additions and 62 deletions
2
tests/fixtures/jsonl-schemas.md
vendored
2
tests/fixtures/jsonl-schemas.md
vendored
|
|
@ -22,7 +22,7 @@
|
|||
| trekbrief-stats | ts, task, slug, mode, interview_turns, review_iterations, brief_quality, research_topics, auto_research, auto_result, intent_approved, project_dir, order_id, order_claimed_at, order_reason | commands/trekbrief.md (orchestrator-emit Phase 7; order_* added by lib/stats/stats-append.mjs) | trekbrief.md:1130-1145 | profile, phase_models, profile_source | none |
|
||||
| trekresearch-stats | ts, question, mode, scope, engine, slug, project_dir, brief_path, dimensions, dimensions_baseline, dimensions_baseline_preserved, effort, conv_turns, empty_turns, unique_sources, agents_local, agents_external, gemini_used, confidence, contradictions, open_questions | lib/stats/stats-append.mjs (trekresearch.md Stats tracking) | trekresearch.md:782-808 | profile, phase_models, parallel_agents, external_research_enabled, profile_source | none |
|
||||
| trekplan-stats | ts, task, mode, slug, brief_path, project_dir, codebase_size, codebase_files, agents_deployed, deep_dives, research_briefs_used, research_scout_used, critic_verdict, guardian_verdict, outcome | lib/stats/stats-append.mjs (trekplan.md Phase 12) | trekplan.md:1115-1135 | profile, phase_models, parallel_agents, profile_source | none |
|
||||
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2891-2913 | profile, phase_models, profile_source | none |
|
||||
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2892-2914 | profile, phase_models, profile_source | none |
|
||||
| trekexecute-stats (autonomy events) | ts, event, known_event, payload | lib/stats/event-emit.mjs `emit()` | event-emit.mjs:78-99 | payload.profile, payload.phase_models, payload.profile_source | none |
|
||||
| trekexecute-stats (PostToolUse Bash) | ts, session_id, command_excerpt, duration_ms, success | hooks/scripts/post-bash-stats.mjs (Bash PostToolUse) | post-bash-stats.mjs:42-54 | none (hook is plugin-level, not profile-aware) | command_excerpt (CWE-212) |
|
||||
| trekreview-stats | ts, slug, verdict, counts (BLOCKER/MAJOR/MINOR/SUGGESTION), reviewed_files_count, mode, duration_ms | lib/stats/stats-append.mjs (trekreview.md Phase 8) | trekreview.md:628-632 | profile, phase_models, profile_source | none |
|
||||
|
|
|
|||
|
|
@ -394,7 +394,9 @@ const SHELL_BLOCK_PLAN = [
|
|||
test('parsePlanVerification: a fenced shell block yields one criterion per command line', () => {
|
||||
const criteria = parsePlanVerification(SHELL_BLOCK_PLAN);
|
||||
assert.deepEqual(criteria.map((c) => c.label), ['V1', 'V2']);
|
||||
assert.equal(criteria[0].command, 'npm test # all green');
|
||||
// A trailing comment is not part of the command: with no shell to drop it,
|
||||
// its words would make the check NOT RUN (release review of 6.3.0, round 6).
|
||||
assert.equal(criteria[0].command, 'npm test');
|
||||
assert.equal(criteria[1].command, 'node --test tests/lib/x.test.mjs');
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -448,8 +448,9 @@ test('review 630 r5 / da7584e7: a check runs without a shell - its words reach t
|
|||
assert.equal(r.status, 'passed', r.output);
|
||||
assert.equal(r.output, '~/x\n*.sh\n$HOME\n', 'no shell expanded a tilde, a glob or a variable');
|
||||
const [missing] = runCriteria([{ label: 'X2', text: 'x', command: 'no-such-runner-r630 --x' }], { cwd: dir, ...PASS_SCREENS });
|
||||
assert.equal(missing.status, 'failed', 'a runner that is not installed is a failure, never a pass');
|
||||
assert.match(missing.output, /ENOENT|not found/i, missing.output);
|
||||
// Round 6: a runner that could not start is NOT RUN (no measurement), never a pass and never a red.
|
||||
assert.equal(missing.status, 'unrunnable', 'a runner that is not installed is not run, never a pass');
|
||||
assert.match(missing.output, /could not start.*ENOENT/i, missing.output);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
|
|
@ -586,3 +587,120 @@ test('review 630 r5 / 41c43285 (operator 2026-10-09): the example plan keeps ste
|
|||
const warned = r.warnings.filter((w) => w.code === 'PLAN_STEP_TESTS_ONLY').map((w) => w.message);
|
||||
assert.equal(warned.length, 2, `the operator kept the warning on steps 1-2: ${JSON.stringify(warned)}`);
|
||||
});
|
||||
|
||||
// ==== round 6 of the release review (2026-10-11), each red on c3641ff ===========
|
||||
|
||||
function gitRepo(prefix) {
|
||||
const dir = mkdtempSync(join(tmpdir(), prefix));
|
||||
for (const args of [['init', '-q'], ['config', 'user.email', 't@example.invalid'], ['config', 'user.name', 't'], ['config', 'commit.gpgsign', 'false']]) {
|
||||
assert.equal(spawnSync('git', args, { cwd: dir }).status, 0);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
const commitAll = (dir, msg) => {
|
||||
assert.equal(spawnSync('git', ['add', '-A'], { cwd: dir }).status, 0);
|
||||
assert.equal(spawnSync('git', ['commit', '-q', '-m', msg], { cwd: dir }).status, 0);
|
||||
return spawnSync('git', ['rev-parse', 'HEAD'], { cwd: dir, encoding: 'utf8' }).stdout.trim();
|
||||
};
|
||||
|
||||
test('review 630 r6 / criteria-runner.mjs:464: a git check stays in the repo too', async () => {
|
||||
const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
||||
for (const cmd of ['git diff /etc/hosts /dev/null', 'git diff ../x tests/a.test.mjs', 'git show HEAD:../x', 'git log -- /etc',
|
||||
'git diff --no-index tests/a tests/b']) {
|
||||
assert.equal(allowedCommand(cmd, { cwd: ROOT }).allowed, false, `${cmd} must not run`);
|
||||
}
|
||||
for (const cmd of ['git diff HEAD~1..HEAD --stat', 'git show HEAD:package.json', 'git log -- lib/', 'git show @~1 --stat']) {
|
||||
const r = allowedCommand(cmd, { cwd: ROOT });
|
||||
assert.equal(r.allowed, true, `the control: ${cmd} (${r.reason})`);
|
||||
}
|
||||
});
|
||||
|
||||
test('review 630 r6 / criteria-runner.mjs:168: a trailing # comment in a shell fence is not part of the command', async () => {
|
||||
const { parsePlanVerification, allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
||||
const plan = '## Verification\n\n```bash\nnpm test # all green\nnode --test tests/a.test.mjs #1\n```\n';
|
||||
assert.deepEqual(parsePlanVerification(plan).map((c) => c.command), ['npm test', 'node --test tests/a.test.mjs']);
|
||||
const example = parsePlanVerification(read('examples/01-add-verbose-flag/plan.md'))[0];
|
||||
assert.equal(example.command, 'node --test tests/');
|
||||
assert.equal(allowedCommand(example.command, { cwd: ROOT }).allowed, true, "the shipped example's first check runs again");
|
||||
});
|
||||
|
||||
test('review 630 r6 / criteria-runner.mjs:466: a glob reaches node --test only where node expands it (Node 21+)', async () => {
|
||||
const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
||||
const old = allowedCommand('node --test tests/lib/*.test.mjs', { cwd: ROOT, nodeMajor: 20 });
|
||||
assert.equal(old.allowed, false);
|
||||
assert.match(old.reason, /21/);
|
||||
assert.equal(allowedCommand('node --test tests/lib/*.test.mjs', { cwd: ROOT, nodeMajor: 22 }).allowed, true);
|
||||
assert.equal(allowedCommand('node --test tests/lib/review-630.test.mjs', { cwd: ROOT, nodeMajor: 20 }).allowed, true, 'the control: no glob, any Node');
|
||||
});
|
||||
|
||||
test('review 630 r6 / plan-schema.mjs:318: a backticked file name with no extension is a path', () => {
|
||||
const step = (files) => `- **Files:** ${files}\n- **Changes:** x\n`;
|
||||
assert.deepEqual(stepTestsOnly(step('`tests/build.test.mjs`, `Makefile`')), { testsOnly: false, files: ['tests/build.test.mjs', 'Makefile'] });
|
||||
assert.deepEqual(stepTestsOnly(step('`spec/a_spec.rb`, `Gemfile`')).testsOnly, false);
|
||||
assert.deepEqual(stepTestsOnly(step('`tests/a.test.mjs` — covers `parse` and `JSON`')).files, ['tests/a.test.mjs'], 'a span inside a description is no path');
|
||||
});
|
||||
|
||||
test('review 630 r6 / plan-schema.mjs:329 (altitude): red proof checks tests-only against the step\'s own diff', async () => {
|
||||
const { proveRed } = await import('../../lib/verification/red-proof.mjs');
|
||||
const work = mkdtempSync(join(tmpdir(), 'r630-diff-'));
|
||||
const repo = gitRepo('r630-diff-repo-');
|
||||
try {
|
||||
writeFileSync(join(repo, 'README.md'), 'x\n');
|
||||
const base = commitAll(repo, 'base');
|
||||
mkdirSync(join(repo, 'tests'));
|
||||
writeFileSync(join(repo, 'tests', 'a.test.mjs'), "import { test } from 'node:test';\ntest('a', () => {});\n");
|
||||
writeFileSync(join(repo, 'Makefile'), 'test:\n\tnode --test tests/\n');
|
||||
// The prose reads only the test (a plain Makefile beside backticked paths is a description word).
|
||||
const plan = join(work, 'plan.md');
|
||||
writeFileSync(plan, '## Implementation Plan\n\n### Step 1: x\n\n- **Files:** `tests/a.test.mjs`, Makefile\n- **Changes:** x\n');
|
||||
const o = { cwd: repo, base, cmd: 'node --test tests/a.test.mjs', tests: ['tests/a.test.mjs'], plan, step: 1 };
|
||||
const r = proveRed(o);
|
||||
assert.equal(r.reason, 'not-red-at-base', JSON.stringify(r));
|
||||
assert.equal(r.tests_only, false, `the step's diff changes Makefile: ${JSON.stringify(r.tests_only_note ?? null)}`);
|
||||
assert.match(r.tests_only_note ?? '', /Makefile/);
|
||||
rmSync(join(repo, 'Makefile'));
|
||||
assert.equal(proveRed(o).tests_only, true, 'the control: a step whose diff holds only its test is tests-only');
|
||||
} finally { rmSync(work, { recursive: true, force: true }); rmSync(repo, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('review 630 r6 / red-proof.mjs:129: exit 127 from the test program itself is a red, not "could not run"', async () => {
|
||||
const { proveRed } = await import('../../lib/verification/red-proof.mjs');
|
||||
const { writeDoneLine } = await import('../../lib/verification/done-line.mjs');
|
||||
const repo = gitRepo('r630-127-');
|
||||
try {
|
||||
writeFileSync(join(repo, 'README.md'), 'x\n');
|
||||
const base = commitAll(repo, 'base');
|
||||
mkdirSync(join(repo, 'tests'));
|
||||
mkdirSync(join(repo, 'bin'));
|
||||
writeFileSync(join(repo, 'tests', 'e2e.sh'), './bin/tool\n');
|
||||
writeFileSync(join(repo, 'bin', 'tool'), '#!/bin/sh\nexit 0\n', { mode: 0o755 });
|
||||
const r = proveRed({ cwd: repo, base, cmd: 'bash tests/e2e.sh', tests: ['tests/e2e.sh'], redOnly: true });
|
||||
assert.equal(r.status, 'proven', JSON.stringify(r));
|
||||
assert.equal(r.base.exit, 127);
|
||||
rmSync(join(repo, 'bin'), { recursive: true });
|
||||
const d = writeDoneLine({ cwd: repo, step: 1, cmd: 'bash tests/e2e.sh', report: join(repo, 'report.md') });
|
||||
assert.equal(d.status, 'failed', `a test that exits 127 failed: ${JSON.stringify(d)}`);
|
||||
} finally { rmSync(repo, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('review 630 r6 / regression-check.mjs:64: a Regression: whose runner cannot start is not-run, not a red suite', async () => {
|
||||
const { checkRegression } = await import('../../lib/verification/regression-check.mjs');
|
||||
const empty = mkdtempSync(join(tmpdir(), 'r630-path-'));
|
||||
const saved = process.env.PATH;
|
||||
try {
|
||||
process.env.PATH = empty;
|
||||
const planText = '## Implementation Plan\n\n### Step 1: x\n\n- **Files:** `src/a.mjs`\n- **Regression:** `vitest run`\n';
|
||||
const r = checkRegression({ planText, step: 1, cwd: ROOT });
|
||||
assert.equal(r.status, 'error', JSON.stringify(r));
|
||||
assert.equal(r.progress_status, 'not-run');
|
||||
assert.equal(r.exit, 2);
|
||||
assert.match(r.output, /could not start/);
|
||||
} finally { process.env.PATH = saved; rmSync(empty, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('review 630 r6 / hooks are ASCII: the lines this release added to pre-bash-executor.mjs', () => {
|
||||
const src = read('hooks/scripts/pre-bash-executor.mjs');
|
||||
const block = src.slice(src.indexOf('Everything this guard would ask about') - 200, src.indexOf('Everything this guard would ask about') + 300);
|
||||
assert.ok(block.includes('Everything this guard would ask about'));
|
||||
assert.doesNotMatch(block, /[^\x00-\x7f]/, 'a hook script is ASCII (~/.claude/CLAUDE.md, Miljø)');
|
||||
});
|
||||
|
|
|
|||
|
|
@ -449,3 +449,16 @@ test('release-check r5 — a test moved into an allowed place after the review i
|
|||
assert.match(r.stdout, /guard\.test\.mjs/);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
// ==== round 6 of the release review of 6.3.0 (2026-10-11) ====================
|
||||
|
||||
test('release-check r6 — a suite that never ends is stopped and read as not measured → 2', () => {
|
||||
const fx = fixture({ testScript: 'node -e "setTimeout(() => {}, 4000)"' });
|
||||
try {
|
||||
const r = mod.releaseCheck({ repo: fx.repo, review: fx.review, suiteTimeoutMs: 1000 });
|
||||
const suite = r.checks.find((c) => c.id === 'suite');
|
||||
assert.equal(suite.status, 'error', JSON.stringify(suite));
|
||||
assert.match(suite.detail, /timeout|killed/i);
|
||||
assert.equal(r.exit, 2);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue