// tests/lib/review-630.test.mjs // The findings of the release review of 6.3.0 (2026-10-10), each held by a // test that failed on f669b87. The guard's and the release check's own tests // live next to their files (tests/commands/trekexecute-coupling.test.mjs, // tests/hooks/guards-review-regressions.test.mjs, tests/scripts/release-check.test.mjs). import { test } from 'node:test'; import { strict as assert } from 'node:assert'; import { spawnSync } from 'node:child_process'; import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { buildPlan } from '../helpers/plan-v18-builder.mjs'; import { validatePlanContent } from '../../lib/validators/plan-validator.mjs'; import { stepTestsOnly } from '../../lib/parsers/plan-schema.mjs'; import { validateFindings } from '../../lib/review/findings-schema.mjs'; import { lintSkill } from '../../lib/skills/skill-lint.mjs'; import * as routing from '../../lib/skills/routing.mjs'; const awaitImportSync = () => routing; const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); const read = (rel) => readFileSync(join(ROOT, rel), 'utf8'); const VALIDATOR = join(ROOT, 'lib', 'validators', 'plan-validator.mjs'); // The review template's own Step R1, appended the Phase 7.2 way. function withTemplateStep(plan, n) { const r1 = /### Step R1:[^\n]*\n[\s\S]*?(?=\n> )/.exec(read('templates/trekreview-template.md')); assert.ok(r1, 'templates/trekreview-template.md carries a ### Step R1'); // Phase 7.2 step 2 also adds the manifest the legacy rule synthesizes (expected_paths ← Files). const appended = r1[0].replace(/^### Step R1:/, `### Step ${n}:`) + '- **Checkpoint:** `git commit -m "fix(review): x [01234567]"`\n' + '- **Manifest:**\n ```yaml\n manifest:\n expected_paths:\n - lib/foo.mjs\n min_file_count: 1\n' + ' commit_message_pattern: "^fix\\\\(review\\\\):"\n bash_syntax_check: []\n forbidden_paths: []\n must_contain: []\n ```\n'; return plan.replace('\n## Test Strategy', `\n${appended}\n## Test Strategy`); } // ---- BLOCKER 1b25ba56: appended review steps hold the 1.8 step rules on any plan test('review 630 / 1b25ba56: on a 1.7 plan, an appended review step that cannot run is refused (--steps-from)', () => { const dir = mkdtempSync(join(tmpdir(), 'r630-')); try { const plan = withTemplateStep(buildPlan({ version: '1.7', design: false }), 3); const p = join(dir, 'plan.md'); writeFileSync(p, plan); const plain = spawnSync(process.execPath, [VALIDATOR, '--strict', '--json', p], { encoding: 'utf8' }); assert.equal(plain.status, 0, 'the control: a 1.7 plan is not held to the 1.8 step rules'); const r = spawnSync(process.execPath, [VALIDATOR, '--strict', '--json', '--steps-from', '3', p], { encoding: 'utf8' }); assert.equal(r.status, 1, `the appended step is held to them: ${r.stdout}`); const codes = JSON.parse(r.stdout).errors.map((e) => e.code); assert.ok(codes.includes('PLAN_STEP_VERIFY_NOT_RUNNABLE') || codes.includes('PLAN_STEP_NO_TEST_FIRST'), codes.join(',')); } finally { rmSync(dir, { recursive: true, force: true }); } }); test('review 630 / 1b25ba56: --steps-from leaves the earlier steps of a 1.7 plan alone', () => { const plan = buildPlan({ version: '1.7', design: false, steps: [{ testFirst: false }, { endToEnd: 'bash tests/e2e.sh' }] }); assert.deepEqual(validatePlanContent(plan, { strict: true, stepsFrom: 3 }).errors, [], 'step 1 has no Test first, and it is not one of the appended steps'); }); test('review 630 / 1b25ba56: Phase 7.2 validates the appended steps with --steps-from', () => { const cmd = read('commands/trekexecute.md'); assert.match(cmd, /plan-validator\.mjs --strict --json --steps-from \{total \+ 1\} "\{plan_path\}"/); }); // ---- 57a5e67c: the wave path names the guard's refusal ------------------------ test('review 630 / 57a5e67c: the wave merge path says what to do when the guard refused because no one can be asked', () => { const cmd = read('commands/trekexecute.md'); const at = cmd.indexOf('**Merge session branches back (SEQUENTIAL, one at a time):**'); const before = cmd.slice(cmd.lastIndexOf('- **`all_completed: true`:**', at), at); assert.match(before, /no one to ask/, 'the guard-refused case is named on the wave path'); assert.match(before, /by hand/, 'and the way on: the operator runs the merge by hand'); assert.match(before, /wave\.json/, 'wave.json is kept'); }); // ---- 2cf29d6f / 727924c0: what counts as a step of only tests ------------------ test('review 630: a comma inside a Files: note is not a path', () => { const body = '- **Files:** `tests/a.test.mjs` (new: covers x, y)\n- **Changes:** add the test\n'; assert.deepEqual(stepTestsOnly(body), { testsOnly: true, files: ['tests/a.test.mjs'] }); }); test('review 630: a data file under specs/ is not a test', () => { const body = '- **Files:** `api/specs/openapi.yaml`\n- **Changes:** add the endpoint\n'; assert.equal(stepTestsOnly(body).testsOnly, false); const fixture = '- **Files:** `tests/fixtures/users.json`\n- **Changes:** add the fixture\n'; assert.equal(stepTestsOnly(fixture).testsOnly, true, 'the control: a fixture under tests/ is a test file'); }); test('review 630: a tests-only End-to-end step is told where End-to-end belongs', () => { const plan = buildPlan({ steps: [{}, { file: 'tests/e2e.sh', test: 'tests/e2e.sh', endToEnd: 'bash tests/e2e.sh' }] }); const w = validatePlanContent(plan, { strict: true }).warnings.find((x) => x.code === 'PLAN_STEP_TESTS_ONLY'); assert.ok(w, 'the warning fires: red proof would reject this step'); assert.match(`${w.message} ${w.hint ?? ''}`, /`End-to-end:` (rides|belongs) on the last step that builds/); assert.match(read('templates/plan-template.md'), /`End-to-end:` (rides|belongs) on the last step that builds/); }); // ---- c4623362: a removed line can still be cited -------------------------------- test('review 630 / c4623362: the out-of-range hint says how to cite a line that no longer exists', () => { const r = validateFindings({ reviewer: 'r', findings: [{ id: 'a'.repeat(40), severity: 'MAJOR', rule_key: 'MISSING_TEST', file: 'package.json', line: 1, brief_ref: 'x', title: 't', recommended_action: 'a', detail: 'the check formerly at package.json:9999 was removed', }] }, { repoRoot: ROOT }); const e = r.errors.find((x) => x.code === 'FINDING_PROSE_LINE_OUT_OF_RANGE'); assert.ok(e, 'the control: the plain citation is still checked'); assert.match(e.hint, /::/); const ok = validateFindings({ reviewer: 'r', findings: [{ id: 'a'.repeat(40), severity: 'MAJOR', rule_key: 'MISSING_TEST', file: 'package.json', line: 1, brief_ref: 'x', title: 't', recommended_action: 'a', detail: 'the check formerly at v6.2.0:package.json:9999 was removed', }] }, { repoRoot: ROOT }); assert.deepEqual(ok.errors, [], 'a revision-qualified citation is not judged against today\'s file'); assert.match(read('commands/trekreview.md'), /::/, 'the re-ask names the form'); }); // ---- 2261016d: a 499-line SKILL.md is under the 500-line limit ------------------- test('review 630 / 2261016d: a SKILL.md of 499 lines ending in a newline passes the size rule', () => { const dir = mkdtempSync(join(tmpdir(), 'r630-skill-')); try { const skill = join(dir, 'test-first'); cpSync(join(ROOT, 'skills', 'test-first'), skill, { recursive: true }); const text = readFileSync(join(skill, 'SKILL.md'), 'utf8').replace(/\n+$/, '\n'); const lines = text.split('\n').length - 1; assert.ok(lines < 499, `the fixture pads the real skill (${lines} lines)`); const padded = text.replace('\n## Credits', `${'\n'.repeat(499 - lines)}\n## Credits`); assert.equal(padded.split('\n').length - 1, 499); writeFileSync(join(skill, 'SKILL.md'), padded); const codes = lintSkill(skill).errors.map((e) => e.code); assert.ok(!codes.includes('SKILL_TOO_LONG'), codes.join(',')); writeFileSync(join(skill, 'SKILL.md'), padded + '\n'); assert.ok(lintSkill(skill).errors.map((e) => e.code).includes('SKILL_TOO_LONG'), 'the control: 500 lines fail'); } finally { rmSync(dir, { recursive: true, force: true }); } }); // ---- cb34c32e: regression-check takes commandEnv from the shared util ------------ test('review 630 / cb34c32e: regression-check takes commandEnv from lib/util/command-env.mjs', () => { // Round 5 (MINOR 0e98d5c7): regression-check has no exec of its own any more; // it runs through the criteria runner's runCommand, which takes commandEnv // from lib/util/command-env.mjs. const src = read('lib/verification/regression-check.mjs'); assert.doesNotMatch(src, /from '\.\/red-proof\.mjs'/); assert.match(src, /runCriteria\(c, o\)/); assert.match(read('lib/verification/criteria-runner.mjs'), /import \{ commandEnv \} from '\.\.\/util\/command-env\.mjs';/); }); // ---- CLAUDE.md names the mode-read exemptions as they are ----------------------- test('review 630: CLAUDE.md counts the mode-read exemptions the launch test holds', () => { const src = read('tests/lib/no-claude-code-launch.test.mjs'); const n = (src.match(/\['mode-read', \d+,/g) || []).length; assert.equal(n, 4, 'the launch test holds four mode-read entries'); assert.match(read('CLAUDE.md'), /four files that only read\s+(?:> )?the session's permission mode/); }); // ---- /security-review, older than v6.2.0: a read-only git that writes a file ---- test('review 630 / security: `git diff --output=` is not a read-only check', async () => { const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs'); assert.equal(allowedCommand('git diff --stat').allowed, true, 'the control: a read-only git check runs'); assert.equal(allowedCommand('git log --output-indicator-new=+ -1').allowed, true, 'an option that only starts with --output is not it'); for (const cmd of ['git diff --output=../../.zshrc', 'git diff --output ../x', 'git log -p --output=x', 'git show HEAD --output=x']) { const r = allowedCommand(cmd); assert.equal(r.allowed, false, `${cmd} writes a file`); assert.match(r.reason, /--output/); } }); // ==== round 2 of the release review (2026-10-10), each red on 0d044f6 =========== test('review 630 r2 / BLOCKER: a quoted, escaped or expanded --output is refused too', async () => { const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs'); for (const cmd of [ 'git diff "--output=/tmp/x"', "git log -p '--output'=x", "git diff '--output=q1'", 'git log "--output" q1', 'git diff --output\\=x', 'git diff --\\output=q1', 'git diff --out{put,x}=y', 'git diff --outp$EMPTY=x', 'git diff --outp?t=x', 'git diff --outpu=x', 'git log --out x', ]) { assert.equal(allowedCommand(cmd).allowed, false, `${cmd} must not reach the shell`); } for (const cmd of ['git log --oneline -1', 'git diff --stat HEAD~1', 'git show HEAD^:package.json', 'git log --format=%h -1', 'git ls-files tests/', 'git log --output-indicator-new=+ -1']) { assert.equal(allowedCommand(cmd).allowed, true, `the control: ${cmd} is a read-only check`); } }); test('review 630 r2: a Files: entry without backticks names every path in it', () => { const body = '- **Files:** tests/a.test.mjs + src/b.mjs\n- **Changes:** x\n'; assert.deepEqual(stepTestsOnly(body), { testsOnly: false, files: ['tests/a.test.mjs', 'src/b.mjs'] }); const and = '- **Files:** tests/a.test.mjs and src/b.mjs\n- **Changes:** x\n'; assert.equal(stepTestsOnly(and).testsOnly, false); const plain = '- **Files:** Makefile\n- **Changes:** x\n'; assert.deepEqual(stepTestsOnly(plain).files, ['Makefile'], 'the control: a bare file name is still a path'); }); test('review 630 r2: a backticked path with parentheses keeps them', () => { const body = '- **Files:** `app/(auth)/login.check.sh` (new)\n- **Changes:** x\n' + '- **Test first:**\n - File: `app/(auth)/login.check.sh` (new)\n - Verifies: y\n'; assert.deepEqual(stepTestsOnly(body), { testsOnly: true, files: ['app/(auth)/login.check.sh'] }); }); test('review 630 r2: --steps-from reads the plan\'s ## Verification for an appended End-to-end step', () => { const plan = buildPlan({ version: '1.7', design: false }); const e2e = '### Step 3: Run it\n\n- **Files:** `src/run.mjs`\n- **Changes:** wire it\n' + '- **Test first:**\n - File: `tests/run.test.mjs` (new)\n - Verifies: it runs\n' + '- **Verify:** `node --test tests/run.test.mjs` → expected: exit 0\n' + '- **End-to-end:** `bash tests/e2e.sh` → expected: the CLI prints ok\n' + '- **Manifest:**\n ```yaml\n manifest:\n expected_paths:\n - src/run.mjs\n min_file_count: 1\n' + ' commit_message_pattern: "^fix"\n bash_syntax_check: []\n forbidden_paths: []\n must_contain: []\n ```\n'; const edited = plan.replace('\n## Test Strategy', `\n${e2e}\n## Test Strategy`); assert.match(edited, /- \[ \] `bash tests\/e2e\.sh`/, 'the fixture lists the command under ## Verification'); const r = validatePlanContent(edited, { strict: true, stepsFrom: 3 }); assert.deepEqual(r.errors.map((e) => e.code), [], JSON.stringify(r.errors)); }); test('review 630 r2: --soft --steps-from warns instead of failing', () => { const plan = withTemplateStep(buildPlan({ version: '1.7', design: false }), 3); const soft = validatePlanContent(plan, { strict: false, stepsFrom: 3 }); assert.equal(soft.valid, true, JSON.stringify(soft.errors)); assert.ok(soft.warnings.some((w) => /VERIFY_NOT_RUNNABLE|TEST_FIRST/.test(w.code)), 'the issue is still reported'); assert.equal(validatePlanContent(plan, { strict: true, stepsFrom: 3 }).valid, false, 'the control: strict fails'); }); test('review 630 r2: regression-check prints the status progress.json takes', async () => { const { checkRegression } = await import('../../lib/verification/regression-check.mjs'); const { PROGRESS_REGRESSION_VALUES } = await import('../../lib/validators/progress-validator.mjs'); const plan = '## Implementation Plan\n\n### Step 1: x\n\n- **Regression:** `npm test`\n'; const blocked = checkRegression({ planText: plan, step: 1, run: () => [{ status: 'blocked', command: 'npm test' }] }); assert.equal(blocked.progress_status, 'unrunnable'); const missing = checkRegression({ planText: plan, step: 9 }); assert.equal(missing.progress_status, 'not-run'); for (const r of [blocked, missing]) assert.ok(PROGRESS_REGRESSION_VALUES.includes(r.progress_status)); assert.match(read('commands/trekexecute.md'), /`progress_status`/, 'the executor writes that field'); }); test('review 630 r2: a skill is found by its name even when an agent has the same name', () => { const dir = mkdtempSync(join(tmpdir(), 'r630-route-')); try { for (const d of ['skills', 'commands', 'agents', 'evals']) cpSync(join(ROOT, d), join(dir, d), { recursive: true }); writeFileSync(join(dir, 'agents', 'debugging.md'), '---\nname: debugging\ndescription: Bakes sourdough loaves and schedules oven time for the shared kitchen rota.\nmodel: opus\n---\n\nBread.\n'); const { checkRouting } = awaitImportSync(); const codes = checkRouting(dir).errors.map((e) => e.code); assert.ok(!codes.includes('ROUTING_UNKNOWN_SKILL'), codes.join(',')); } finally { rmSync(dir, { recursive: true, force: true }); } }); test('review 630 r2: rank() does not rebuild the document vectors per call', () => { const { buildCorpus, rank } = awaitImportSync(); const corpus = buildCorpus([ { id: 'a', name: 'alpha', description: 'parses plans and validates steps' }, { id: 'b', name: 'beta', description: 'runs tests and reports failures' }, ]); let calls = 0; const idf = corpus.idf; corpus.idf = (t) => { calls++; return idf(t); }; rank('validates plans', corpus); rank('reports failures', corpus); assert.ok(calls <= 4, `idf was called ${calls} times for two two-word prompts`); }); test('review 630 r2 (operator 2026-10-10): Phase 7.2 says plainly that template-form steps stop with a re-plan', () => { const cmd = read('commands/trekexecute.md'); assert.match(cmd, /Today the loop runs only for steps written as\s+runnable plan steps/); assert.match(cmd, /re-plan from the review" — the\s+usual outcome until a later order/); }); // ==== round 3 of the release review (2026-10-10), each red on 96cb977 =========== test('review 630 r3 / BLOCKER: no runner takes an option that writes a file or loads code', async () => { const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs'); for (const cmd of [ 'node --test --test-reporter=tap --test-reporter-destination=/tmp/x', 'node --test --import=data:text/javascript,1', 'node --test "--import=data:text/javascript,import(\'node:fs\')"', 'node --test --require ./evil.cjs', 'node --test --test-reporter=./evil-reporter.mjs', 'node --test --test-reporter ./evil-reporter.mjs', 'jest --json --outputFile=/tmp/x', 'jest --config=evil.js', 'vitest --outputFile=/tmp/x', 'vitest run --reporter=./evil.mjs', 'pytest --junitxml=/tmp/x', 'pytest --basetemp=/tmp/anydir', 'pytest -p evil_plugin', 'python -m pytest --junitxml=/tmp/x', 'uv run pytest --basetemp=/tmp/x', 'go test -o /tmp/x', 'go test -exec /tmp/evil ./...', 'cargo test --target-dir=/tmp/x', 'npm test -- --test-reporter-destination=/tmp/x', 'npm test --script-shell=/tmp/evil', 'npm run verify -- --outputFile=/tmp/x', 'make test SHELL=/tmp/evil', 'make test -f /tmp/evil.mk', 'bash tests/x.sh "$(id)"', ]) { assert.equal(allowedCommand(cmd, { cwd: ROOT }).allowed, false, `${cmd} must not reach the shell`); } for (const cmd of [ 'npm test', 'npm test -- tests/lib/criteria-runner.test.mjs', 'node --test tests/lib/criteria-runner.test.mjs', 'node --test tests/lib/*.test.mjs', 'node --test --test-reporter=tap tests/lib/review-630.test.mjs', 'node --test --test-reporter dot', 'node --test --test-name-pattern=review tests/lib/review-630.test.mjs', 'vitest run', 'jest --ci', 'pytest -q', 'pytest -x -q tests/', 'python3 -m pytest tests/', 'uv run pytest', 'cargo test', 'cargo test --release', 'go test ./...', 'go test -run TestX -count=1 ./pkg', 'make test', 'bash tests/fixtures/criteria-exit-0.sh', 'bash tests/fixtures/criteria-exit-0.sh --fast', ]) { const r = allowedCommand(cmd, { cwd: ROOT }); assert.equal(r.allowed, true, `the control: ${cmd} is a test run (${r.reason})`); } }); test('review 630 r3: a restore the guard refuses is never recorded as reverted', async () => { const { PROGRESS_SIMPLIFY_VALUES } = await import('../../lib/validators/progress-validator.mjs'); assert.ok(PROGRESS_SIMPLIFY_VALUES.includes('revert-refused'), 'progress.json can say the restore was refused'); assert.match(read('docs/HANDOVER-CONTRACTS.md'), /`applied \\| clean \\| reverted \\| revert-refused \\| not-run`/); const cmd = read('commands/trekexecute.md'); const onFailure = cmd.slice(cmd.indexOf('**`On failure: revert`** and **`On failure: retry`**'), cmd.indexOf('**`On failure: skip`**')); assert.match(onFailure, /guard refuses the restore/, 'the step revert says what happens when the guard refuses it'); const simplify = cmd.slice(cmd.indexOf('2. **`/simplify`**'), cmd.indexOf('3. **`/trekreview`**')); assert.match(simplify, /simplify = "revert-refused"/); assert.match(simplify, /never `"reverted"`/); }); test('review 630 r3: a fixture document under tests/ is a test file, and "e.g." is not a path', () => { const step = (files) => `- **Files:** ${files}\n- **Changes:** x\n`; assert.equal(stepTestsOnly(step('`tests/a.test.mjs`, `tests/fixtures/sample.md`')).testsOnly, true); assert.deepEqual(stepTestsOnly(step('tests/a.test.mjs (new), e.g. the fixture tests/fixtures/x.json')), { testsOnly: true, files: ['tests/a.test.mjs', 'tests/fixtures/x.json'] }); assert.equal(stepTestsOnly(step('`docs/specs/api.md`')).testsOnly, false, 'a document outside tests/ is still not a test'); assert.deepEqual(stepTestsOnly(step('`./tests/a.test.mjs`')).files, ['tests/a.test.mjs'], './ is stripped'); }); test('review 630 r3: the plan-file sweep and the tests-only check both drop a leading ./', async () => { // Round 3 had the sweep read Files: through the tests-only reader; the // operator reversed that on 2026-10-11 (the K6 sweep keeps v6.2.0's reader, // round 5). What the round-3 finding named still holds: `./` is dropped by both. const { plannedPaths } = await import('../../lib/verification/plan-scope-sweep.mjs'); const plan = '## Implementation Plan\n\n### Step 1: x\n\n- **Files:** `./src/a.mjs`, `./tests/b.test.mjs`\n- **Changes:** x\n'; const body = plan.slice(plan.indexOf('### Step 1')); assert.deepEqual(plannedPaths(plan), ['src/a.mjs', 'tests/b.test.mjs']); assert.deepEqual(stepTestsOnly(body).files, ['src/a.mjs', 'tests/b.test.mjs']); }); test('review 630 r3: --steps-from refuses an appended step with no Verify: on a 1.8 plan too', () => { const plan = buildPlan({}); const step = '### Step 3: Fix it\n\n- **Files:** `src/fix.mjs`, `tests/fix.test.mjs`\n- **Changes:** fix it\n' + '- **Test first:**\n - File: `tests/fix.test.mjs` (new)\n - Verifies: it is fixed\n- **On failure:** escalate\n'; const edited = plan.replace('\n## Test Strategy', `\n${step}\n## Test Strategy`); assert.ok(edited.includes('### Step 3: Fix it'), 'the fixture appends the step'); const r = validatePlanContent(edited, { strict: true, stepsFrom: 3 }); assert.ok(r.errors.some((e) => e.code === 'PLAN_STEP_VERIFY_NOT_RUNNABLE' && /Step 3/.test(e.message)), JSON.stringify(r.errors.map((e) => e.code))); }); test('review 630 r3: the plan validator hands its parsed fields to the tests-only check', async () => { const schema = await import('../../lib/parsers/plan-schema.mjs'); assert.equal(typeof schema.stepTestsOnlyFromFields, 'function'); const body = '- **Files:** `tests/a.test.mjs`\n- **Changes:** x\n'; assert.deepEqual(schema.stepTestsOnlyFromFields(schema.parseStepFields(body)), stepTestsOnly(body)); assert.match(read('lib/validators/plan-validator.mjs'), /stepTestsOnlyFromFields\(f\)/); }); test('review 630 r3: skill frontmatter - a value starting with # is a hazard, an escaped backslash closes a quote', async () => { const { plainScalarHazards, readFrontmatterFields } = await import('../../lib/skills/skill-frontmatter.mjs'); assert.ok(plainScalarHazards('#1 helper. Use when testing.').length > 0, 'YAML reads "#..." as a comment: the value is null'); const r = readFrontmatterFields('---\nname: x\ndescription: "Path C:\\\\"\n---\nbody\n'); assert.equal(r.fields.description?.value, 'Path C:\\', JSON.stringify(r)); const open = readFrontmatterFields('---\nname: x\ndescription: "a \\"\n---\nbody\n'); assert.ok(JSON.stringify(open).includes('unclosed double quote'), 'a quote escaped by one backslash is still open'); }); test('review 630 r3: routing reads a kind tag only from its own table, never Object.prototype', () => { const dir = mkdtempSync(join(tmpdir(), 'r630-proto-')); try { for (const d of ['skills', 'commands', 'agents', 'evals']) cpSync(join(ROOT, d), join(dir, d), { recursive: true }); const c = join(dir, 'evals', 'zz-proto'); cpSync(join(ROOT, 'evals', 'skill-test-first-quiet-1'), c, { recursive: true }); writeFileSync(join(c, 'prompt.md'), '---\ntags: [constructor, owner=nobody]\n---\n\nWhat time is it?\n'); const { checkRouting } = awaitImportSync(); const r = checkRouting(dir); assert.ok(!r.errors.some((e) => String(e.message ?? '').includes('zz-proto') || e.case === 'zz-proto'), JSON.stringify(r.errors)); } finally { rmSync(dir, { recursive: true, force: true }); } }); // ==== round 4 of the release review (2026-10-10), each red on 41ea8fc =========== test('review 630 r4: the Files: reader keeps every path the field names, and only those', async () => { const { fieldPaths } = await import('../../lib/parsers/plan-schema.mjs'); const cases = [ ['`src/a.mjs` (fixture: `tests/fixtures/a.json`)', ['src/a.mjs', 'tests/fixtures/a.json']], ['`tests/a.test.mjs`, `src/a.mjs`.', ['tests/a.test.mjs', 'src/a.mjs']], ['`src/cli.mjs`: add the flag; `src/b.mjs`.', ['src/cli.mjs', 'src/b.mjs']], ['`tests/a.test.mjs` — covers parse, format', ['tests/a.test.mjs']], ['docs/ and README.md', ['docs/', 'README.md']], ['app/(auth)/x.ts', ['app/(auth)/x.ts']], ['src/*.mjs (all of them)', ['src/*.mjs']], ['notes for v6.3.0, src/x.mjs', ['src/x.mjs']], ['Makefile', ['Makefile']], ['tests/a.test.mjs + src/b.mjs', ['tests/a.test.mjs', 'src/b.mjs']], ]; for (const [text, want] of cases) assert.deepEqual(fieldPaths(text), want, text); const body = '- **Files:** `tests/a.test.mjs`, `src/a.mjs`.\n- **Changes:** x\n'; assert.equal(stepTestsOnly(body).testsOnly, false, 'a step that changes src/a.mjs is not tests-only'); const { plannedPaths } = await import('../../lib/verification/plan-scope-sweep.mjs'); const plan = '## Implementation Plan\n\n### Step 1: x\n\n- **Files:** `src/cli.mjs`: add the flag; `src/b.mjs`.\n- **Changes:** x\n'; assert.deepEqual(plannedPaths(plan), ['src/b.mjs', 'src/cli.mjs'], 'K6 plans what v6.2.0 planned'); }); test('review 630 r4: a quoted frontmatter value may carry a YAML comment', async () => { const { readFrontmatterFields } = await import('../../lib/skills/skill-frontmatter.mjs'); const r = readFrontmatterFields('---\nname: "debugging" # pinned\ndescription: \'Use it.\' # note\n---\nbody\n'); assert.equal(r.fields.name?.value, 'debugging', JSON.stringify(r)); assert.equal(r.fields.description?.value, 'Use it.', JSON.stringify(r)); const bad = readFrontmatterFields('---\nname: "debugging" trailing\n---\nbody\n'); assert.ok(JSON.stringify(bad).includes('unclosed double quote') || bad.fields.name?.value !== 'debugging', 'text after the closing quote that is not a comment is not accepted silently'); }); test('review 630 r4: no tilde word, and nothing after `make test`', async () => { const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs'); for (const cmd of ['node --test ~/x.test.mjs', 'node --test --test-name-pattern=~x', 'git log ~/x', 'git diff ~root/x', 'make test install', 'make test deploy']) { assert.equal(allowedCommand(cmd, { cwd: ROOT }).allowed, false, `${cmd} must not reach the shell`); } for (const cmd of ['git diff --stat HEAD~1', 'git show HEAD~2:package.json', 'make test']) { assert.equal(allowedCommand(cmd, { cwd: ROOT }).allowed, true, `the control: ${cmd}`); } }); // ==== round 5 of the release review (2026-10-11), each red on d37e538 =========== // Operator 2026-10-11: no more patching of the same two places. A check runs // with no shell (its words reach the runner as written), and the K6 sweep has // its own v6.2.0 reader again; the shared reader stays for the tests-only check. const PASS_SCREENS = { allow: () => ({ allowed: true, reason: '' }), screen: () => ({ allowed: true, rule: '' }) }; test('review 630 r5 / da7584e7: a check runs without a shell - its words reach the runner as written', async () => { const { runCriteria } = await import('../../lib/verification/criteria-runner.mjs'); const dir = mkdtempSync(join(tmpdir(), 'r630-noshell-')); try { mkdirSync(join(dir, 'tests')); writeFileSync(join(dir, 'tests', 'args.sh'), 'printf "%s\\n" "$@"\n'); const [r] = runCriteria([{ label: 'X1', text: 'x', command: 'bash tests/args.sh ~/x *.sh $HOME' }], { cwd: dir, ...PASS_SCREENS }); assert.equal(r.status, 'passed', r.output); assert.equal(r.output, '~/x\n*.sh\n$HOME\n', 'no shell expanded a tilde, a glob or a variable'); const [missing] = runCriteria([{ label: 'X2', text: 'x', command: 'no-such-runner-r630 --x' }], { cwd: dir, ...PASS_SCREENS }); // Round 6: a runner that could not start is NOT RUN (no measurement), never a pass and never a red. assert.equal(missing.status, 'unrunnable', 'a runner that is not installed is not run, never a pass'); assert.match(missing.output, /could not start.*ENOENT/i, missing.output); } finally { rmSync(dir, { recursive: true, force: true }); } }); test('review 630 r5 / c25f8704: a glob cannot climb out of tests/ or the repo', async () => { const { runCriteria, allowedCommand } = await import('../../lib/verification/criteria-runner.mjs'); const base = mkdtempSync(join(tmpdir(), 'r630-escape-')); const repo = join(base, 'repo'); try { mkdirSync(join(repo, 'tests'), { recursive: true }); mkdirSync(join(repo, 'outside')); writeFileSync(join(repo, 'outside', 'evil.sh'), 'echo ESCAPED > escaped-bash.txt\n'); writeFileSync(join(repo, 'tests', 'a.test.mjs'), "import { test } from 'node:test';\ntest('in', () => {});\n"); writeFileSync(join(base, 'outside.test.mjs'), `import { test } from 'node:test';\nimport { writeFileSync } from 'node:fs';\ntest('out', () => { writeFileSync(${JSON.stringify(join(base, 'escaped-node.txt'))}, 'x'); });\n`); const results = runCriteria([ { label: 'E1', text: 'x', command: 'bash tests/.?/outside/evil.sh' }, { label: 'E2', text: 'x', command: 'node --test .*' }, ], { cwd: repo }); assert.equal(existsSync(join(repo, 'escaped-bash.txt')), false, `bash ran a script outside tests/: ${JSON.stringify(results[0])}`); assert.equal(existsSync(join(base, 'escaped-node.txt')), false, `node --test ran a test file outside the repo: ${JSON.stringify(results[1])}`); assert.notEqual(results[0].status, 'passed'); for (const cmd of ['node --test ../outside.test.mjs', 'node --test tests/../../outside.test.mjs', 'node --test /tmp/x.test.mjs', 'pytest ../tests', 'go test ../...', 'npm test -- ../x.test.mjs', 'jest /abs/x.test.js', 'vitest run ../x', 'bash tests/*.sh', 'pytest tests/test_*.py', 'jest src/*.test.js', 'npm test -- tests/*.mjs']) { assert.equal(allowedCommand(cmd, { cwd: ROOT }).allowed, false, `${cmd} must not run`); } for (const cmd of ['node --test tests/lib/*.test.mjs', 'node --test tests/**/*.test.mjs', 'go test ./...', '/bin/bash tests/fixtures/criteria-exit-0.sh', 'node --test tests/', 'pytest tests/', 'git diff HEAD~1..HEAD --stat']) { const r = allowedCommand(cmd, { cwd: ROOT }); assert.equal(r.allowed, true, `the control: ${cmd} (${r.reason})`); } } finally { rmSync(base, { recursive: true, force: true }); } }); test('review 630 r5 / 8f3cf84d: a word starting with @ is refused (pytest reads @file as more arguments)', async () => { const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs'); for (const cmd of ['pytest @tests/args.txt', 'python -m pytest @tests/args.txt', 'python3 -m pytest -q @a', 'uv run pytest @a', 'node --test @a', 'jest @a']) { const r = allowedCommand(cmd, { cwd: ROOT }); assert.equal(r.allowed, false, `${cmd} must not run`); assert.match(r.reason, /@/); } }); test('review 630 r5 / da7584e7: every place that runs a check runs it with no shell', () => { for (const rel of ['lib/verification/criteria-runner.mjs', 'lib/verification/regression-check.mjs', 'lib/verification/red-proof.mjs', 'lib/verification/done-line.mjs', 'lib/util/test-lock.mjs']) { const src = read(rel); assert.doesNotMatch(src, /spawnSync\(\s*['"](?:\/bin\/)?(?:ba)?sh['"]/, `${rel} starts a shell`); assert.doesNotMatch(src, /shell:\s*true/, `${rel} asks spawn for a shell`); if (rel !== 'lib/verification/criteria-runner.mjs') assert.match(src, /\brunCommand\b/, `${rel} runs through criteria-runner's runCommand`); } assert.doesNotMatch(read('lib/verification/regression-check.mjs'), /function exec\(/, 'regression-check has no exec of its own (MINOR 0e98d5c7)'); }); test('review 630 r5 / da7584e7: the done line runs its command with no shell', async () => { // sh reads `**` as `*` (one level); node --test expands its own pattern at // every depth. Only with no shell does the top-level test run. const { writeDoneLine } = await import('../../lib/verification/done-line.mjs'); const repo = mkdtempSync(join(tmpdir(), 'r630-done-')); try { mkdirSync(join(repo, 'tests', 'sub'), { recursive: true }); writeFileSync(join(repo, 'tests', 'sub', 'deep.test.mjs'), "import { test } from 'node:test';\ntest('deep', () => {});\n"); writeFileSync(join(repo, 'tests', 'top.test.mjs'), `import { test } from 'node:test';\nimport { writeFileSync } from 'node:fs';\ntest('top', () => { writeFileSync(${JSON.stringify(join(repo, 'top-ran.txt'))}, 'x'); });\n`); for (const args of [['init', '-q'], ['-c', 'user.email=t@example.invalid', '-c', 'user.name=t', 'commit', '-q', '--allow-empty', '-m', 'x']]) { assert.equal(spawnSync('git', args, { cwd: repo }).status, 0); } const r = writeDoneLine({ cwd: repo, step: 1, cmd: 'node --test tests/**/*.test.mjs', report: join(repo, 'report.md') }); assert.equal(r.status, 'written', JSON.stringify(r)); assert.equal(existsSync(join(repo, 'top-ran.txt')), true, 'the pattern reached node as written'); } finally { rmSync(repo, { recursive: true, force: true }); } }); test('review 630 r5 / ff80c7ef 5dc53c7e 02cea4aa: the K6 sweep plans what v6.2.0 planned, with its own reader', async () => { const { plannedPaths } = await import('../../lib/verification/plan-scope-sweep.mjs'); const plan = (files) => `## Implementation Plan\n\n### Step 1: x\n\n- **Files:** ${files}\n- **Changes:** x\n`; const cases = [ ['`src/a.mjs` — replaces the helper in lib/util.mjs and touches config.json; see Node.js docs', ['src/a.mjs']], ['`src/cli.mjs` — follows the pattern in lib/', ['src/cli.mjs']], ['.gitignore, src/a.mjs', ['.gitignore', 'src/a.mjs']], ['src\\cli.mjs', ['src/cli.mjs']], ['`src/a.mjs`/`src/b.mjs`', ['src/a.mjs', 'src/b.mjs']], ['`src/a.mjs`—adds', ['src/a.mjs']], ['`src/cli.mjs`: add the flag; `src/b.mjs`.', ['src/b.mjs', 'src/cli.mjs']], ]; for (const [files, want] of cases) assert.deepEqual(plannedPaths(plan(files)), want, files); assert.doesNotMatch(read('lib/verification/plan-scope-sweep.mjs'), /import \{[^}]*\bfieldPaths\b/, 'K6 does not read Files: through the tests-only reader'); }); test('review 630 r5 / a4c8005d 02cea4aa: the tests-only check takes only what passes a path test', () => { const step = (files) => `- **Files:** ${files}\n- **Changes:** x\n`; const cases = [ ['`tests/parse.test.mjs` (asserts `parse()` throws)', true, ['tests/parse.test.mjs']], ['`tests/a.test.mjs` — covers pass/fail', true, ['tests/a.test.mjs']], ['`tests/a.test.mjs` — covers JSON, YAML', true, ['tests/a.test.mjs']], ['`tests/a.test.mjs` — calls `parser.parse()` twice', true, ['tests/a.test.mjs']], ['`tests/a.test.mjs`, .gitignore', false, ['tests/a.test.mjs', '.gitignore']], ['tests/a.test.mjs, .gitignore', false, ['tests/a.test.mjs', '.gitignore']], ['`tests/a.test.mjs`/`src/b.mjs`', false, ['tests/a.test.mjs', 'src/b.mjs']], ['`tests\\a.test.mjs`', true, ['tests/a.test.mjs']], ['tests\\a.test.mjs, src\\cli.mjs', false, ['tests/a.test.mjs', 'src/cli.mjs']], ['Makefile.', false, ['Makefile']], ]; for (const [files, testsOnly, want] of cases) assert.deepEqual(stepTestsOnly(step(files)), { testsOnly, files: want }, files); }); test('review 630 r5 / e1ff583a f582b9c2: frontmatter reads trailing space and a comment after a plain value as YAML does', async () => { const { readFrontmatterFields } = await import('../../lib/skills/skill-frontmatter.mjs'); const r = readFrontmatterFields('---\nname: "debugging" \nmodel: opus # operator pin\neffort: high # pinned\ntags: [a] \n' + 'allowed-tools: [Read, Grep] # read only\ndescription: | # folded below\n Use when x.\n---\nbody\n'); assert.deepEqual(r.hazards, [], JSON.stringify(r.hazards)); assert.equal(r.fields.name.value, 'debugging'); assert.equal(r.fields.model.value, 'opus'); assert.equal(r.fields.effort.value, 'high'); assert.deepEqual(r.fields.tags.value, ['a']); assert.deepEqual(r.fields['allowed-tools'].value, ['Read', 'Grep']); assert.equal(r.fields.description.value, 'Use when x.\n'); // The control: a description is the text Claude Code routes on, so a comment // that cuts it is still a hazard (quote it), and so is a value that is only a comment. for (const fm of ['description: Use when x happens #1.', 'description: Use when x # note', 'name: # x']) { assert.ok(readFrontmatterFields(`---\n${fm}\n---\nbody\n`).hazards.length > 0, fm); } }); test('review 630 r5 / 41c43285 (operator 2026-10-09): the example plan keeps steps 1-2, and step 2 no longer says it must fail', () => { const plan = read('examples/01-add-verbose-flag/plan.md'); const heading = plan.match(/^### Step 2: .*$/m)[0]; assert.doesNotMatch(heading, /must FAIL/i, heading); assert.match(plan.slice(plan.indexOf(heading)), /should PASS today/, 'the step body still says it passes today'); const progress = JSON.parse(read('examples/01-add-verbose-flag/progress.json')); const title = JSON.stringify(progress).match(/"title":"(Add stability test[^"]*)"/)?.[1]; assert.equal(`### Step 2: ${title}`, heading, 'progress.json carries the same step title'); const r = validatePlanContent(plan, { strict: false }); const warned = r.warnings.filter((w) => w.code === 'PLAN_STEP_TESTS_ONLY').map((w) => w.message); assert.equal(warned.length, 2, `the operator kept the warning on steps 1-2: ${JSON.stringify(warned)}`); }); // ==== round 6 of the release review (2026-10-11), each red on c3641ff =========== function gitRepo(prefix) { const dir = mkdtempSync(join(tmpdir(), prefix)); for (const args of [['init', '-q'], ['config', 'user.email', 't@example.invalid'], ['config', 'user.name', 't'], ['config', 'commit.gpgsign', 'false']]) { assert.equal(spawnSync('git', args, { cwd: dir }).status, 0); } return dir; } const commitAll = (dir, msg) => { assert.equal(spawnSync('git', ['add', '-A'], { cwd: dir }).status, 0); assert.equal(spawnSync('git', ['commit', '-q', '-m', msg], { cwd: dir }).status, 0); return spawnSync('git', ['rev-parse', 'HEAD'], { cwd: dir, encoding: 'utf8' }).stdout.trim(); }; test('review 630 r6 / criteria-runner.mjs:464: a git check stays in the repo too', async () => { const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs'); for (const cmd of ['git diff /etc/hosts /dev/null', 'git diff ../x tests/a.test.mjs', 'git show HEAD:../x', 'git log -- /etc', 'git diff --no-index tests/a tests/b']) { assert.equal(allowedCommand(cmd, { cwd: ROOT }).allowed, false, `${cmd} must not run`); } for (const cmd of ['git diff HEAD~1..HEAD --stat', 'git show HEAD:package.json', 'git log -- lib/', 'git show @~1 --stat']) { const r = allowedCommand(cmd, { cwd: ROOT }); assert.equal(r.allowed, true, `the control: ${cmd} (${r.reason})`); } }); test('review 630 r6 / criteria-runner.mjs:168: a trailing # comment in a shell fence is not part of the command', async () => { const { parsePlanVerification, allowedCommand } = await import('../../lib/verification/criteria-runner.mjs'); const plan = '## Verification\n\n```bash\nnpm test # all green\nnode --test tests/a.test.mjs #1\n```\n'; assert.deepEqual(parsePlanVerification(plan).map((c) => c.command), ['npm test', 'node --test tests/a.test.mjs']); const example = parsePlanVerification(read('examples/01-add-verbose-flag/plan.md'))[0]; assert.equal(example.command, 'node --test tests/'); assert.equal(allowedCommand(example.command, { cwd: ROOT }).allowed, true, "the shipped example's first check runs again"); }); test('review 630 r6 / criteria-runner.mjs:466: a glob reaches node --test only where node expands it (Node 21+)', async () => { const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs'); const old = allowedCommand('node --test tests/lib/*.test.mjs', { cwd: ROOT, nodeMajor: 20 }); assert.equal(old.allowed, false); assert.match(old.reason, /21/); assert.equal(allowedCommand('node --test tests/lib/*.test.mjs', { cwd: ROOT, nodeMajor: 22 }).allowed, true); assert.equal(allowedCommand('node --test tests/lib/review-630.test.mjs', { cwd: ROOT, nodeMajor: 20 }).allowed, true, 'the control: no glob, any Node'); }); test('review 630 r6 / plan-schema.mjs:318: a backticked file name with no extension is a path', () => { const step = (files) => `- **Files:** ${files}\n- **Changes:** x\n`; assert.deepEqual(stepTestsOnly(step('`tests/build.test.mjs`, `Makefile`')), { testsOnly: false, files: ['tests/build.test.mjs', 'Makefile'] }); assert.deepEqual(stepTestsOnly(step('`spec/a_spec.rb`, `Gemfile`')).testsOnly, false); assert.deepEqual(stepTestsOnly(step('`tests/a.test.mjs` — covers `parse` and `JSON`')).files, ['tests/a.test.mjs'], 'a span inside a description is no path'); }); test('review 630 r6 / plan-schema.mjs:329 (altitude): red proof checks tests-only against the step\'s own diff', async () => { const { proveRed } = await import('../../lib/verification/red-proof.mjs'); const work = mkdtempSync(join(tmpdir(), 'r630-diff-')); const repo = gitRepo('r630-diff-repo-'); try { writeFileSync(join(repo, 'README.md'), 'x\n'); const base = commitAll(repo, 'base'); mkdirSync(join(repo, 'tests')); writeFileSync(join(repo, 'tests', 'a.test.mjs'), "import { test } from 'node:test';\ntest('a', () => {});\n"); writeFileSync(join(repo, 'Makefile'), 'test:\n\tnode --test tests/\n'); // The prose reads only the test (a plain Makefile beside backticked paths is a description word). const plan = join(work, 'plan.md'); writeFileSync(plan, '## Implementation Plan\n\n### Step 1: x\n\n- **Files:** `tests/a.test.mjs`, Makefile\n- **Changes:** x\n'); const o = { cwd: repo, base, cmd: 'node --test tests/a.test.mjs', tests: ['tests/a.test.mjs'], plan, step: 1 }; const r = proveRed(o); assert.equal(r.reason, 'not-red-at-base', JSON.stringify(r)); assert.equal(r.tests_only, false, `the step's diff changes Makefile: ${JSON.stringify(r.tests_only_note ?? null)}`); assert.match(r.tests_only_note ?? '', /Makefile/); rmSync(join(repo, 'Makefile')); assert.equal(proveRed(o).tests_only, true, 'the control: a step whose diff holds only its test is tests-only'); } finally { rmSync(work, { recursive: true, force: true }); rmSync(repo, { recursive: true, force: true }); } }); test('review 630 r6 / red-proof.mjs:129: exit 127 from the test program itself is a red, not "could not run"', async () => { const { proveRed } = await import('../../lib/verification/red-proof.mjs'); const { writeDoneLine } = await import('../../lib/verification/done-line.mjs'); const repo = gitRepo('r630-127-'); try { writeFileSync(join(repo, 'README.md'), 'x\n'); const base = commitAll(repo, 'base'); mkdirSync(join(repo, 'tests')); mkdirSync(join(repo, 'bin')); writeFileSync(join(repo, 'tests', 'e2e.sh'), './bin/tool\n'); writeFileSync(join(repo, 'bin', 'tool'), '#!/bin/sh\nexit 0\n', { mode: 0o755 }); const r = proveRed({ cwd: repo, base, cmd: 'bash tests/e2e.sh', tests: ['tests/e2e.sh'], redOnly: true }); assert.equal(r.status, 'proven', JSON.stringify(r)); assert.equal(r.base.exit, 127); rmSync(join(repo, 'bin'), { recursive: true }); const d = writeDoneLine({ cwd: repo, step: 1, cmd: 'bash tests/e2e.sh', report: join(repo, 'report.md') }); assert.equal(d.status, 'failed', `a test that exits 127 failed: ${JSON.stringify(d)}`); } finally { rmSync(repo, { recursive: true, force: true }); } }); test('review 630 r6 / regression-check.mjs:64: a Regression: whose runner cannot start is not-run, not a red suite', async () => { const { checkRegression } = await import('../../lib/verification/regression-check.mjs'); const empty = mkdtempSync(join(tmpdir(), 'r630-path-')); const saved = process.env.PATH; try { process.env.PATH = empty; const planText = '## Implementation Plan\n\n### Step 1: x\n\n- **Files:** `src/a.mjs`\n- **Regression:** `vitest run`\n'; const r = checkRegression({ planText, step: 1, cwd: ROOT }); assert.equal(r.status, 'error', JSON.stringify(r)); assert.equal(r.progress_status, 'not-run'); assert.equal(r.exit, 2); assert.match(r.output, /could not start/); } finally { process.env.PATH = saved; rmSync(empty, { recursive: true, force: true }); } }); test('review 630 r6 / hooks are ASCII: the lines this release added to pre-bash-executor.mjs', () => { const src = read('hooks/scripts/pre-bash-executor.mjs'); const block = src.slice(src.indexOf('Everything this guard would ask about') - 200, src.indexOf('Everything this guard would ask about') + 300); assert.ok(block.includes('Everything this guard would ask about')); assert.doesNotMatch(block, /[^\x00-\x7f]/, 'a hook script is ASCII (~/.claude/CLAUDE.md, Miljø)'); });