// tests/lib/retterunde.test.mjs // The fix round before 6.3.0 (order 10), the points that are behaviour, not // prose. Each test was red before its fix. // 1 A step whose `Files:` are all tests can never be proven red: the // behaviour it tests is already at its start point (voyage-lekerepo // trial 3, 2026-10-09, finding 1). (a) the validator warns // PLAN_STEP_TESTS_ONLY; (c) red-proof says `tests_only` on its // `not-red-at-base`, so the executor sends the step up to the plan instead // of into fix rounds that cannot succeed. One reading: stepTestsOnly. // 3 The example plan's `On failure:` lines open with a K11 word. // 5 The criteria runner's commands never see NODE_TEST_CONTEXT: a nested // `node --test` would report to the parent and exit 0 on a red test. import { test } from 'node:test'; import { strict as assert } from 'node:assert'; import { spawnSync } from 'node:child_process'; import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { validatePlan, validatePlanContent } from '../../lib/validators/plan-validator.mjs'; import { parseStepFields, sliceSteps, stepTestsOnly } from '../../lib/parsers/plan-schema.mjs'; import { proveRed } from '../../lib/verification/red-proof.mjs'; import { runCriteria } from '../../lib/verification/criteria-runner.mjs'; import { buildPlan } from '../helpers/plan-v18-builder.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); const ROOT = join(HERE, '..', '..'); const codes = (list) => list.map((e) => e.code); // --- 1 (a): the validator ----------------------------------------------------- // Step 2 changes only tests for behaviour step 1 delivered, as in the trial. const TESTS_ONLY_PLAN = buildPlan({ steps: [{}, { file: 'tests/e2e.sh', test: 'tests/e2e-guard.test.mjs', endToEnd: 'bash tests/e2e.sh' }], }); test('1a: a step whose Files are all tests gives PLAN_STEP_TESTS_ONLY, a warning on that step', () => { const r = validatePlanContent(TESTS_ONLY_PLAN); const w = r.warnings.filter((x) => x.code === 'PLAN_STEP_TESTS_ONLY'); assert.equal(w.length, 1, codes(r.warnings).join(',')); assert.match(w[0].message, /Step 2/); assert.ok(!codes(r.errors).includes('PLAN_STEP_TESTS_ONLY'), 'a warning, never an error'); assert.equal(r.valid, true, codes(r.errors).join(',')); }); test('1a: a step with code and its test gives no PLAN_STEP_TESTS_ONLY (as before)', () => { const r = validatePlanContent(buildPlan()); assert.ok(!codes(r.warnings).includes('PLAN_STEP_TESTS_ONLY'), codes(r.warnings).join(',')); }); test('1a: the reading - Files all tests (by path or as the Test first file) or not; no Files = no warning', () => { const body = (files, tf = 'tests/x.test.mjs') => [ '### Step 1: x', '', ...(files === null ? [] : [`- **Files:** ${files}`]), '- **Test first:**', ` - File: \`${tf}\` (new)`, ' - Verifies: x', ].join('\n'); assert.equal(stepTestsOnly(body('`tests/e2e.sh`, `tests/e2e-guard.sh`')).testsOnly, true); assert.equal(stepTestsOnly(body('`src/a.test.ts` (new file), `lib/b.spec.js`, `pkg/c_test.go`')).testsOnly, true); assert.equal(stepTestsOnly(body('`__tests__/a.js`, `test/b.js`')).testsOnly, true); // The Test first file counts as a test wherever it lives. assert.equal(stepTestsOnly(body('`scripts/check.sh`', 'scripts/check.sh')).testsOnly, true); assert.equal(stepTestsOnly(body('`src/cli.mjs`, `tests/cli.test.mjs`')).testsOnly, false); assert.equal(stepTestsOnly(body('`src/cli.mjs`')).testsOnly, false); assert.equal(stepTestsOnly(body(null)).testsOnly, false); assert.deepEqual(stepTestsOnly(body('`tests/e2e.sh`, `tests/e2e-guard.sh`')).files, ['tests/e2e.sh', 'tests/e2e-guard.sh']); }); test('1a: a plain path beside a backticked one is still read - a step that also changes code is not tests-only', () => { const body = (files) => ['### Step 1: x', '', `- **Files:** ${files}`].join('\n'); const mixed = stepTestsOnly(body('src/cli.mjs (modify), `tests/x.test.mjs` (new)')); assert.deepEqual(mixed.files, ['src/cli.mjs', 'tests/x.test.mjs']); assert.equal(mixed.testsOnly, false); const list = stepTestsOnly(['### Step 1: x', '', '- **Files:**', ' - `tests/x.test.mjs` (new)', ' - src/cli.mjs'].join('\n')); assert.equal(list.testsOnly, false, JSON.stringify(list.files)); }); test('1a: a document under spec/ or specs/ is not a test for this reading', () => { const body = (files) => ['### Step 1: x', '', `- **Files:** ${files}`].join('\n'); assert.equal(stepTestsOnly(body('`docs/specs/api.md`, `specs/001-auth/plan.md`')).testsOnly, false); assert.equal(stepTestsOnly(body('`spec/cli_spec.rb`')).testsOnly, true); }); // --- 1 (c): red-proof says tests_only on not-red-at-base ---------------------- function sh(cwd, cmd, args) { const r = spawnSync(cmd, args, { cwd, encoding: 'utf8' }); assert.equal(r.status, 0, `${cmd} ${args.join(' ')}: ${r.stderr}`); return r.stdout.trim(); } const SUM_TEST = [ "import { test } from 'node:test';", "import { strict as assert } from 'node:assert';", "import { sum } from '../lib/sum.mjs';", "test('sum adds', () => { assert.equal(sum(1, 2), 3); });", '', ].join('\n'); /** The behaviour is already at the start point; the step adds only its test. */ function greenAtBase() { const dir = mkdtempSync(join(tmpdir(), 'retterunde-red-')); sh(dir, 'git', ['init', '-q']); sh(dir, 'git', ['config', 'user.email', 't@example.invalid']); sh(dir, 'git', ['config', 'user.name', 'test']); mkdirSync(join(dir, 'lib')); writeFileSync(join(dir, 'lib', 'sum.mjs'), 'export function sum(a, b) { return a + b; }\n'); sh(dir, 'git', ['add', 'lib/sum.mjs']); sh(dir, 'git', ['commit', '-q', '-m', 'base']); mkdirSync(join(dir, 'tests')); writeFileSync(join(dir, 'tests', 'sum.test.mjs'), SUM_TEST); return { dir, base: sh(dir, 'git', ['rev-parse', 'HEAD']) }; } function planWithStep2Files(dir, files) { const p = join(dir, 'plan.local.md'); writeFileSync(p, buildPlan({ steps: [{}, { file: files, test: 'tests/sum.test.mjs', endToEnd: 'bash tests/e2e.sh' }] })); return p; } const CMD = 'node --test tests/sum.test.mjs'; test('1c: not-red-at-base on a step whose Files are all tests says tests_only: true (red-only and full)', () => { const { dir, base } = greenAtBase(); try { const plan = planWithStep2Files(dir, 'tests/sum.test.mjs'); for (const redOnly of [true, false]) { const r = proveRed({ cwd: dir, base, cmd: CMD, tests: ['tests/sum.test.mjs'], redOnly, plan, step: 2 }); assert.equal(r.reason, 'not-red-at-base', JSON.stringify(r)); assert.equal(r.tests_only, true, JSON.stringify(r)); } const cli = spawnSync(process.execPath, [join(ROOT, 'lib', 'verification', 'red-proof.mjs'), '--cwd', dir, '--base', base, '--cmd', CMD, '--tests', 'tests/sum.test.mjs', '--plan', plan, '--step', '2', '--json'], { encoding: 'utf8' }); assert.equal(cli.status, 1, cli.stdout + cli.stderr); assert.equal(JSON.parse(cli.stdout).tests_only, true); // Text mode names the field too (the step helper reads it there). const text = spawnSync(process.execPath, [join(ROOT, 'lib', 'verification', 'red-proof.mjs'), '--red-only', '--cwd', dir, '--base', base, '--cmd', CMD, '--tests', 'tests/sum.test.mjs', '--plan', plan, '--step', '2'], { encoding: 'utf8' }); assert.match(text.stdout, /tests_only: true/); // --step without --plan is a usage error, never silently dropped. const lone = spawnSync(process.execPath, [join(ROOT, 'lib', 'verification', 'red-proof.mjs'), '--cwd', dir, '--base', base, '--cmd', CMD, '--tests', 'tests/sum.test.mjs', '--step', '2'], { encoding: 'utf8' }); assert.equal(lone.status, 2, lone.stderr); } finally { rmSync(dir, { recursive: true, force: true }); } }); test('1c: not-red-at-base on a step that also changes code says tests_only: false (fix rounds, as before)', () => { const { dir, base } = greenAtBase(); try { const plan = planWithStep2Files(dir, 'lib/sum.mjs'); const r = proveRed({ cwd: dir, base, cmd: CMD, tests: ['tests/sum.test.mjs'], plan, step: 2 }); assert.equal(r.reason, 'not-red-at-base'); assert.equal(r.tests_only, false, JSON.stringify(r)); // Without --plan nothing new is said: the field is absent. const bare = proveRed({ cwd: dir, base, cmd: CMD, tests: ['tests/sum.test.mjs'] }); assert.equal(bare.reason, 'not-red-at-base'); assert.equal('tests_only' in bare, false); } finally { rmSync(dir, { recursive: true, force: true }); } }); // --- 3: the example plan reads with the K11 words ----------------------------- test('3: the example plan gives no PLAN_STEP_ON_FAILURE_UNKNOWN', () => { const r = validatePlan(join(ROOT, 'examples', '01-add-verbose-flag', 'plan.md')); assert.deepEqual(r.warnings.filter((w) => w.code === 'PLAN_STEP_ON_FAILURE_UNKNOWN').map((w) => w.message), []); }); // --- 5: the criteria runner strips NODE_TEST_CONTEXT --------------------------- test('5: a nested `node --test` on a failing test is red even when NODE_TEST_CONTEXT is set', () => { const dir = mkdtempSync(join(tmpdir(), 'retterunde-ctx-')); const saved = process.env.NODE_TEST_CONTEXT; try { mkdirSync(join(dir, 'tests')); writeFileSync(join(dir, 'tests', 'fail.test.mjs'), [ "import { test } from 'node:test';", "import { strict as assert } from 'node:assert';", "test('fails', () => { assert.equal(1, 2); });", '', ].join('\n')); process.env.NODE_TEST_CONTEXT = 'child-v8'; const [r] = runCriteria([{ label: 'C1', text: 'x', command: 'node --test tests/fail.test.mjs' }], { cwd: dir }); assert.equal(r.status, 'failed', JSON.stringify(r)); assert.notEqual(r.exitCode, 0); } finally { if (saved === undefined) delete process.env.NODE_TEST_CONTEXT; else process.env.NODE_TEST_CONTEXT = saved; rmSync(dir, { recursive: true, force: true }); } }); // Operator 2026-10-09: the example's verbose-content test moves into the step // that wires the verbose output; steps 1-2 (golden capture and the stability // harness: tests of today's behaviour, never red) keep the warning, and a step // type for them is a wave 3 question. test('1: the example plan warns only on its two golden/stability steps; the verbose test sits with the code', () => { const r = validatePlan(join(ROOT, 'examples', '01-add-verbose-flag', 'plan.md')); const w = r.warnings.filter((x) => x.code === 'PLAN_STEP_TESTS_ONLY').map((x) => x.message.match(/Step (\d+)/)[1]); assert.deepEqual(w, ['1', '2']); const text = readFileSync(join(ROOT, 'examples', '01-add-verbose-flag', 'plan.md'), 'utf8'); const step = sliceSteps(text).find((s) => /tests\/cli-verbose-flag\.test\.mjs/.test(parseStepFields(s.body).get('Files')?.text ?? '')); assert.ok(step, 'a step lists tests/cli-verbose-flag.test.mjs in Files'); assert.match(parseStepFields(step.body).get('Files').text, /src\/commands\/login\.mjs/); // The run record follows the plan: one entry per step. const progress = JSON.parse(readFileSync(join(ROOT, 'examples', '01-add-verbose-flag', 'progress.json'), 'utf8')); const n = r.parsed.steps.length; assert.equal(progress.total_steps, n); assert.equal(progress.current_step, n); assert.deepEqual(progress.steps.map((s) => s.n), Array.from({ length: n }, (_, i) => i + 1)); }); // Guard against a reading of the example that counts nothing: it has steps. test('3: the example plan is read (six or more steps)', () => { const text = readFileSync(join(ROOT, 'examples', '01-add-verbose-flag', 'plan.md'), 'utf8'); assert.ok(validatePlanContent(text).parsed.steps.length >= 6); });