fix: round 2 of the release review of 6.3.0
Round 2 (v6.2.0..0d044f6, BLOCK: 2 BLOCKER, 11 MAJOR, 1 MINOR), each fixed behind a test that failed first. - The git runner checks only words /bin/sh passes through unchanged (no quote, backslash, $, glob or brace) and refuses --output and its abbreviations: round 1's --output refusal was bypassed by quoting. - fieldPaths: backticked paths are kept as written, parentheses and all; in plain text every path-like word counts, and a parenthesised note does not. - plan-validator --steps-from reads the plan's ## Verification for an appended End-to-end step, and warns instead of failing under --soft. - regression-check prints progress_status, the value progress.json takes; /trekexecute copies it. - release-check: a plugin in a subdirectory is checked in its own directory, the suite is not run when a cheaper check already makes the answer exit 2, and a bad argument or a crash is exit 2. - Skill routing looks a skill up among skills only, and builds the corpus vectors once. - /trekexecute Phase 7.2 says plainly that remediation steps in the review template's own form stop with a re-plan; making them runnable is a later order (operator 2026-10-10). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
0d044f6b6f
commit
ed87cc668d
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
10 changed files with 263 additions and 70 deletions
2
tests/fixtures/jsonl-schemas.md
vendored
2
tests/fixtures/jsonl-schemas.md
vendored
|
|
@ -22,7 +22,7 @@
|
|||
| trekbrief-stats | ts, task, slug, mode, interview_turns, review_iterations, brief_quality, research_topics, auto_research, auto_result, intent_approved, project_dir, order_id, order_claimed_at, order_reason | commands/trekbrief.md (orchestrator-emit Phase 7; order_* added by lib/stats/stats-append.mjs) | trekbrief.md:1130-1145 | profile, phase_models, profile_source | none |
|
||||
| trekresearch-stats | ts, question, mode, scope, engine, slug, project_dir, brief_path, dimensions, dimensions_baseline, dimensions_baseline_preserved, effort, conv_turns, empty_turns, unique_sources, agents_local, agents_external, gemini_used, confidence, contradictions, open_questions | lib/stats/stats-append.mjs (trekresearch.md Stats tracking) | trekresearch.md:782-808 | profile, phase_models, parallel_agents, external_research_enabled, profile_source | none |
|
||||
| trekplan-stats | ts, task, mode, slug, brief_path, project_dir, codebase_size, codebase_files, agents_deployed, deep_dives, research_briefs_used, research_scout_used, critic_verdict, guardian_verdict, outcome | lib/stats/stats-append.mjs (trekplan.md Phase 12) | trekplan.md:1115-1135 | profile, phase_models, parallel_agents, profile_source | none |
|
||||
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2867-2889 | profile, phase_models, profile_source | none |
|
||||
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2877-2899 | profile, phase_models, profile_source | none |
|
||||
| trekexecute-stats (autonomy events) | ts, event, known_event, payload | lib/stats/event-emit.mjs `emit()` | event-emit.mjs:78-99 | payload.profile, payload.phase_models, payload.profile_source | none |
|
||||
| trekexecute-stats (PostToolUse Bash) | ts, session_id, command_excerpt, duration_ms, success | hooks/scripts/post-bash-stats.mjs (Bash PostToolUse) | post-bash-stats.mjs:42-54 | none (hook is plugin-level, not profile-aware) | command_excerpt (CWE-212) |
|
||||
| trekreview-stats | ts, slug, verdict, counts (BLOCKER/MAJOR/MINOR/SUGGESTION), reviewed_files_count, mode, duration_ms | lib/stats/stats-append.mjs (trekreview.md Phase 8) | trekreview.md:628-632 | profile, phase_models, profile_source | none |
|
||||
|
|
|
|||
|
|
@ -16,6 +16,9 @@ import { validatePlanContent } from '../../lib/validators/plan-validator.mjs';
|
|||
import { stepTestsOnly } from '../../lib/parsers/plan-schema.mjs';
|
||||
import { validateFindings } from '../../lib/review/findings-schema.mjs';
|
||||
import { lintSkill } from '../../lib/skills/skill-lint.mjs';
|
||||
import * as routing from '../../lib/skills/routing.mjs';
|
||||
|
||||
const awaitImportSync = () => routing;
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const read = (rel) => readFileSync(join(ROOT, rel), 'utf8');
|
||||
|
|
@ -164,3 +167,101 @@ test('review 630 / security: `git diff --output=<path>` is not a read-only check
|
|||
assert.match(r.reason, /--output/);
|
||||
}
|
||||
});
|
||||
|
||||
// ==== round 2 of the release review (2026-10-10), each red on 0d044f6 ===========
|
||||
|
||||
test('review 630 r2 / BLOCKER: a quoted, escaped or expanded --output is refused too', async () => {
|
||||
const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
||||
for (const cmd of [
|
||||
'git diff "--output=/tmp/x"', "git log -p '--output'=x", "git diff '--output=q1'", 'git log "--output" q1',
|
||||
'git diff --output\\=x', 'git diff --\\output=q1', 'git diff --out{put,x}=y', 'git diff --outp$EMPTY=x',
|
||||
'git diff --outp?t=x', 'git diff --outpu=x', 'git log --out x',
|
||||
]) {
|
||||
assert.equal(allowedCommand(cmd).allowed, false, `${cmd} must not reach the shell`);
|
||||
}
|
||||
for (const cmd of ['git log --oneline -1', 'git diff --stat HEAD~1', 'git show HEAD^:package.json', 'git log --format=%h -1',
|
||||
'git ls-files tests/', 'git log --output-indicator-new=+ -1']) {
|
||||
assert.equal(allowedCommand(cmd).allowed, true, `the control: ${cmd} is a read-only check`);
|
||||
}
|
||||
});
|
||||
|
||||
test('review 630 r2: a Files: entry without backticks names every path in it', () => {
|
||||
const body = '- **Files:** tests/a.test.mjs + src/b.mjs\n- **Changes:** x\n';
|
||||
assert.deepEqual(stepTestsOnly(body), { testsOnly: false, files: ['tests/a.test.mjs', 'src/b.mjs'] });
|
||||
const and = '- **Files:** tests/a.test.mjs and src/b.mjs\n- **Changes:** x\n';
|
||||
assert.equal(stepTestsOnly(and).testsOnly, false);
|
||||
const plain = '- **Files:** Makefile\n- **Changes:** x\n';
|
||||
assert.deepEqual(stepTestsOnly(plain).files, ['Makefile'], 'the control: a bare file name is still a path');
|
||||
});
|
||||
|
||||
test('review 630 r2: a backticked path with parentheses keeps them', () => {
|
||||
const body = '- **Files:** `app/(auth)/login.check.sh` (new)\n- **Changes:** x\n' +
|
||||
'- **Test first:**\n - File: `app/(auth)/login.check.sh` (new)\n - Verifies: y\n';
|
||||
assert.deepEqual(stepTestsOnly(body), { testsOnly: true, files: ['app/(auth)/login.check.sh'] });
|
||||
});
|
||||
|
||||
test('review 630 r2: --steps-from reads the plan\'s ## Verification for an appended End-to-end step', () => {
|
||||
const plan = buildPlan({ version: '1.7', design: false });
|
||||
const e2e = '### Step 3: Run it\n\n- **Files:** `src/run.mjs`\n- **Changes:** wire it\n' +
|
||||
'- **Test first:**\n - File: `tests/run.test.mjs` (new)\n - Verifies: it runs\n' +
|
||||
'- **Verify:** `node --test tests/run.test.mjs` → expected: exit 0\n' +
|
||||
'- **End-to-end:** `bash tests/e2e.sh` → expected: the CLI prints ok\n' +
|
||||
'- **Manifest:**\n ```yaml\n manifest:\n expected_paths:\n - src/run.mjs\n min_file_count: 1\n' +
|
||||
' commit_message_pattern: "^fix"\n bash_syntax_check: []\n forbidden_paths: []\n must_contain: []\n ```\n';
|
||||
const edited = plan.replace('\n## Test Strategy', `\n${e2e}\n## Test Strategy`);
|
||||
assert.match(edited, /- \[ \] `bash tests\/e2e\.sh`/, 'the fixture lists the command under ## Verification');
|
||||
const r = validatePlanContent(edited, { strict: true, stepsFrom: 3 });
|
||||
assert.deepEqual(r.errors.map((e) => e.code), [], JSON.stringify(r.errors));
|
||||
});
|
||||
|
||||
test('review 630 r2: --soft --steps-from warns instead of failing', () => {
|
||||
const plan = withTemplateStep(buildPlan({ version: '1.7', design: false }), 3);
|
||||
const soft = validatePlanContent(plan, { strict: false, stepsFrom: 3 });
|
||||
assert.equal(soft.valid, true, JSON.stringify(soft.errors));
|
||||
assert.ok(soft.warnings.some((w) => /VERIFY_NOT_RUNNABLE|TEST_FIRST/.test(w.code)), 'the issue is still reported');
|
||||
assert.equal(validatePlanContent(plan, { strict: true, stepsFrom: 3 }).valid, false, 'the control: strict fails');
|
||||
});
|
||||
|
||||
test('review 630 r2: regression-check prints the status progress.json takes', async () => {
|
||||
const { checkRegression } = await import('../../lib/verification/regression-check.mjs');
|
||||
const { PROGRESS_REGRESSION_VALUES } = await import('../../lib/validators/progress-validator.mjs');
|
||||
const plan = '## Implementation Plan\n\n### Step 1: x\n\n- **Regression:** `npm test`\n';
|
||||
const blocked = checkRegression({ planText: plan, step: 1, run: () => [{ status: 'blocked', command: 'npm test' }] });
|
||||
assert.equal(blocked.progress_status, 'unrunnable');
|
||||
const missing = checkRegression({ planText: plan, step: 9 });
|
||||
assert.equal(missing.progress_status, 'not-run');
|
||||
for (const r of [blocked, missing]) assert.ok(PROGRESS_REGRESSION_VALUES.includes(r.progress_status));
|
||||
assert.match(read('commands/trekexecute.md'), /`progress_status`/, 'the executor writes that field');
|
||||
});
|
||||
|
||||
test('review 630 r2: a skill is found by its name even when an agent has the same name', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'r630-route-'));
|
||||
try {
|
||||
for (const d of ['skills', 'commands', 'agents', 'evals']) cpSync(join(ROOT, d), join(dir, d), { recursive: true });
|
||||
writeFileSync(join(dir, 'agents', 'debugging.md'),
|
||||
'---\nname: debugging\ndescription: Bakes sourdough loaves and schedules oven time for the shared kitchen rota.\nmodel: opus\n---\n\nBread.\n');
|
||||
const { checkRouting } = awaitImportSync();
|
||||
const codes = checkRouting(dir).errors.map((e) => e.code);
|
||||
assert.ok(!codes.includes('ROUTING_UNKNOWN_SKILL'), codes.join(','));
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('review 630 r2: rank() does not rebuild the document vectors per call', () => {
|
||||
const { buildCorpus, rank } = awaitImportSync();
|
||||
const corpus = buildCorpus([
|
||||
{ id: 'a', name: 'alpha', description: 'parses plans and validates steps' },
|
||||
{ id: 'b', name: 'beta', description: 'runs tests and reports failures' },
|
||||
]);
|
||||
let calls = 0;
|
||||
const idf = corpus.idf;
|
||||
corpus.idf = (t) => { calls++; return idf(t); };
|
||||
rank('validates plans', corpus);
|
||||
rank('reports failures', corpus);
|
||||
assert.ok(calls <= 4, `idf was called ${calls} times for two two-word prompts`);
|
||||
});
|
||||
|
||||
test('review 630 r2 (operator 2026-10-10): Phase 7.2 says plainly that template-form steps stop with a re-plan', () => {
|
||||
const cmd = read('commands/trekexecute.md');
|
||||
assert.match(cmd, /Today the loop runs only for steps written as\s+runnable plan steps/);
|
||||
assert.match(cmd, /re-plan from the review" — the\s+usual outcome until a later order/);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -85,55 +85,57 @@ const DOORS = `### Doors
|
|||
function fixture({
|
||||
suiteExit = 0, doors = DOORS, verdict = 'ALLOW', writeReview = true,
|
||||
reviewSha = 'work', afterRelease = null, dirty = false, version = '1.0.1',
|
||||
testScript = null, files = {}, attributes = null,
|
||||
testScript = null, files = {}, attributes = null, sub = null,
|
||||
} = {}) {
|
||||
const base = mkdtempSync(join(tmpdir(), 'release-check-'));
|
||||
const repo = join(base, 'repo');
|
||||
mkdirSync(join(repo, '.claude-plugin'), { recursive: true });
|
||||
// sub: the plugin lives in a subdirectory of a larger repo (release review of 6.3.0, round 2).
|
||||
const dir = sub ? join(repo, sub) : repo;
|
||||
mkdirSync(join(dir, '.claude-plugin'), { recursive: true });
|
||||
git(repo, 'init', '-q', '-b', 'main');
|
||||
git(repo, 'config', 'user.email', 't@example.invalid');
|
||||
git(repo, 'config', 'user.name', 't');
|
||||
git(repo, 'config', 'commit.gpgsign', 'false');
|
||||
|
||||
const pkg = (v) => JSON.stringify({ name: 'fx', version: v, scripts: { test: testScript ?? `node -e "process.exit(${suiteExit})"` } }, null, 2) + '\n';
|
||||
writeFileSync(join(repo, 'package.json'), pkg('1.0.0'));
|
||||
writeFileSync(join(repo, '.claude-plugin', 'plugin.json'), JSON.stringify({ name: 'fx', version: '1.0.0' }) + '\n');
|
||||
writeFileSync(join(repo, 'README.md'), '# fx\n\n\n\nText.\n');
|
||||
writeFileSync(join(repo, 'CHANGELOG.md'), '# Changelog\n\n## v1.0.0 — 2026-10-01 — first\n\n- first\n');
|
||||
writeFileSync(join(repo, 'a.mjs'), 'export const a = 1;\n');
|
||||
for (const [rel, text] of Object.entries(files)) writeFileSync(join(repo, rel), text);
|
||||
writeFileSync(join(dir, 'package.json'), pkg('1.0.0'));
|
||||
writeFileSync(join(dir, '.claude-plugin', 'plugin.json'), JSON.stringify({ name: 'fx', version: '1.0.0' }) + '\n');
|
||||
writeFileSync(join(dir, 'README.md'), '# fx\n\n\n\nText.\n');
|
||||
writeFileSync(join(dir, 'CHANGELOG.md'), '# Changelog\n\n## v1.0.0 — 2026-10-01 — first\n\n- first\n');
|
||||
writeFileSync(join(dir, 'a.mjs'), 'export const a = 1;\n');
|
||||
for (const [rel, text] of Object.entries(files)) writeFileSync(join(dir, rel), text);
|
||||
if (attributes) writeFileSync(join(repo, '.git', 'info', 'attributes'), attributes);
|
||||
git(repo, 'add', '-A');
|
||||
git(repo, 'commit', '-q', '-m', 'feat: first');
|
||||
writeFileSync(join(repo, 'a.mjs'), 'export const a = 2;\n');
|
||||
git(repo, 'add', 'a.mjs');
|
||||
git(repo, 'commit', '-q', '-m', 'fix: the work');
|
||||
const workSha = git(repo, 'rev-parse', 'HEAD');
|
||||
git(dir, 'add', '-A');
|
||||
git(dir, 'commit', '-q', '-m', 'feat: first');
|
||||
writeFileSync(join(dir, 'a.mjs'), 'export const a = 2;\n');
|
||||
git(dir, 'add', 'a.mjs');
|
||||
git(dir, 'commit', '-q', '-m', 'fix: the work');
|
||||
const workSha = git(dir, 'rev-parse', 'HEAD');
|
||||
|
||||
if (version !== null) {
|
||||
writeFileSync(join(repo, 'package.json'), pkg(version));
|
||||
writeFileSync(join(repo, '.claude-plugin', 'plugin.json'), JSON.stringify({ name: 'fx', version }) + '\n');
|
||||
writeFileSync(join(dir, 'package.json'), pkg(version));
|
||||
writeFileSync(join(dir, '.claude-plugin', 'plugin.json'), JSON.stringify({ name: 'fx', version }) + '\n');
|
||||
} else {
|
||||
writeFileSync(join(repo, '.claude-plugin', 'plugin.json'), JSON.stringify({ name: 'fx' }) + '\n');
|
||||
writeFileSync(join(repo, 'package.json'), JSON.stringify({ name: 'fx', scripts: { test: 'node -e "process.exit(0)"' } }) + '\n');
|
||||
writeFileSync(join(dir, '.claude-plugin', 'plugin.json'), JSON.stringify({ name: 'fx' }) + '\n');
|
||||
writeFileSync(join(dir, 'package.json'), JSON.stringify({ name: 'fx', scripts: { test: 'node -e "process.exit(0)"' } }) + '\n');
|
||||
}
|
||||
writeFileSync(join(repo, 'README.md'), `# fx\n\n\n\nText.\n`);
|
||||
writeFileSync(join(repo, 'CHANGELOG.md'),
|
||||
writeFileSync(join(dir, 'README.md'), `# fx\n\n\n\nText.\n`);
|
||||
writeFileSync(join(dir, 'CHANGELOG.md'),
|
||||
`# Changelog\n\n## v${version ?? '9.9.9'} — 2026-10-08 — fixture\n\n${doors ?? ''}\n### Fixed\n\n- the work\n\n## v1.0.0 — 2026-10-01 — first\n\n- first\n`);
|
||||
git(repo, 'add', '-A');
|
||||
git(repo, 'commit', '-q', '-m', 'chore(release): fixture');
|
||||
git(dir, 'add', '-A');
|
||||
git(dir, 'commit', '-q', '-m', 'chore(release): fixture');
|
||||
|
||||
if (afterRelease) {
|
||||
for (const [rel, text] of Object.entries(afterRelease)) writeFileSync(join(repo, rel), text);
|
||||
git(repo, 'add', '-A');
|
||||
git(repo, 'commit', '-q', '-m', 'after the review');
|
||||
for (const [rel, text] of Object.entries(afterRelease)) writeFileSync(join(dir, rel), text);
|
||||
git(dir, 'add', '-A');
|
||||
git(dir, 'commit', '-q', '-m', 'after the review');
|
||||
}
|
||||
if (dirty) writeFileSync(join(repo, 'a.mjs'), 'export const a = 3;\n');
|
||||
if (dirty) writeFileSync(join(dir, 'a.mjs'), 'export const a = 3;\n');
|
||||
|
||||
const review = join(base, 'review.md');
|
||||
const sha = reviewSha === 'work' ? workSha : reviewSha;
|
||||
if (writeReview) writeFileSync(review, reviewText({ verdict, sha }));
|
||||
return { base, repo, review, cleanup: () => rmSync(base, { recursive: true, force: true }) };
|
||||
return { base, repo: dir, review, cleanup: () => rmSync(base, { recursive: true, force: true }) };
|
||||
}
|
||||
|
||||
function run(fx, extra = []) {
|
||||
|
|
@ -360,3 +362,29 @@ test('release-check — a red suite is red when the check itself runs under node
|
|||
assert.match(r.stdout, /npm test exited/);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
// ---- round 2 of the release review (2026-10-10) --------------------------------
|
||||
|
||||
test('release-check — a plugin in a subdirectory of a larger repo is checked there → 0', () => {
|
||||
const fx = fixture({ sub: 'voyage' });
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 0, r.stdout + r.stderr);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — no review: the suite is not run, exit 2 at once', () => {
|
||||
const fx = fixture({ writeReview: false, testScript: 'node -e "require(\'fs\').writeFileSync(\'suite-ran\', \'\')"' });
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 2, r.stdout + r.stderr);
|
||||
assert.equal(existsSync(join(fx.repo, 'suite-ran')), false, 'the suite did not run');
|
||||
assert.match(r.stdout, /suite\s+not run/);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — a flag with no value is exit 2, not 1', () => {
|
||||
const r = spawnSync(process.execPath, [CHECK, '--repo'], { encoding: 'utf8' });
|
||||
assert.equal(r.status, 2, r.stdout + r.stderr);
|
||||
assert.match(r.stdout + r.stderr, /not measured|NOT MEASURED/i);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue