// tests/lib/test-first-multi-file.test.mjs // Plan step 3 (files-backticks trial 4): below plan_version 1.9 every test file // on the `Test first:` `File:` line is read, not only the first backticked // span. 6.3.1 read the first span alone, so a step whose `Files:` were its two // `Test first:` files was not seen as tests-only when the second file is not a // test by its path. One reading: testFirstFiles (plan-schema.mjs), used by // stepTestsOnly; the executor and the step helper say the same in prose. import { test } from 'node:test'; import { strict as assert } from 'node:assert'; import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import * as m from '../../lib/parsers/plan-schema.mjs'; import { validatePlanContent } from '../../lib/validators/plan-validator.mjs'; import { isTestPath } from '../../lib/util/test-path.mjs'; import { buildPlan } from '../helpers/plan-v18-builder.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); const ROOT = join(HERE, '..', '..'); /** A 1.8-shaped step section: `Files:` and the `Test first:` lines given. */ const step = (files, testFirst) => [ '### Step 1: x', '', `- **Files:** ${files}`, '- **Changes:** x', '- **Test first:**', ...testFirst, '- **Verify:** `node --test tests/a.test.mjs` -> expected: exit 0', ].join('\n'); const TWO = step('`tests/a.test.mjs`, `checks/smoke.mjs`', [ ' - File: `tests/a.test.mjs`, `checks/smoke.mjs` (new)', ' - Verifies: both checks run', ]); const testFirstFiles = (body, version) => { assert.equal(typeof m.testFirstFiles, 'function', 'plan-schema.mjs exports testFirstFiles'); return m.testFirstFiles(m.parseStepFields(body), version); }; test('a 1.8 step whose Files are its two Test first files is tests-only', () => { // checks/smoke.mjs is not a test by its path: only the Test first line makes it one. assert.equal(m.stepTestsOnly(TWO).testsOnly, true, JSON.stringify(m.stepTestsOnly(TWO))); }); test('testFirstFiles returns every path on the File: line, below 1.9 and without a version', () => { assert.deepEqual(testFirstFiles(TWO, '1.8'), ['tests/a.test.mjs', 'checks/smoke.mjs']); assert.deepEqual(testFirstFiles(TWO, null), ['tests/a.test.mjs', 'checks/smoke.mjs']); assert.deepEqual(testFirstFiles(TWO), ['tests/a.test.mjs', 'checks/smoke.mjs']); }); test('testFirstFiles: a call, an option or a URL on the File: line is not read; paths are normalised and deduplicated', () => { const body = step('`tests/a.test.mjs`', [ ' - File: `tests/a.test.mjs`, `parse()`, `--watch`, `https://example.invalid/x.mjs`, `./tests/a.test.mjs`', ' - Verifies: x', ]); assert.deepEqual(testFirstFiles(body, '1.8'), ['tests/a.test.mjs']); }); test('testFirstFiles: a Verifies: line, or a span before File: on the line, is never read', () => { const body = step('`tests/a.test.mjs`', [ ' - see `lib/before.mjs` File: `tests/a.test.mjs`', ' - Verifies: `lib/other.mjs` returns `src/x.mjs`', ]); assert.deepEqual(testFirstFiles(body, '1.8'), ['tests/a.test.mjs']); const none = step('`tests/a.test.mjs`', [' - Verifies: `tests/a.test.mjs` runs']); assert.deepEqual(testFirstFiles(none, '1.8'), []); }); test('a code file beside the test on the File: line still makes the step not tests-only when Files has more', () => { const body = step('`tests/a.test.mjs`, `src/cli.mjs`', [ ' - File: `tests/a.test.mjs`, `parse()` (new)', ' - Verifies: x', ]); assert.equal(m.stepTestsOnly(body).testsOnly, false); }); // ---- plan_version 1.9 (plan step 5) ---------------------------------------- // From 1.9 a `Test first:` `File:` line is every line that opens with `File:` // (after an optional list marker and `**`), backticked or not, and its rest is // one entry read by entryPaths: the leading backticked paths are the test // files, the rest is description. The validator puts the same grammar on it. const PATH_CODES = ['PLAN_STEP_PATH_NOT_FIRST', 'PLAN_STEP_PATH_NOT_A_PATH']; // A plan of the given version whose first step has `Files:` `files` and the // `Test first:` `File:` line `fileLine` (both verbatim). const planWith = (version, files, fileLine) => buildPlan({ version, steps: [{ filesRaw: files, testFirstRaw: fileLine }, { endToEnd: 'bash tests/e2e.sh' }], }); test('1.9: every leading backticked path on the File: line is a test file', () => { const body = step('`tests/a.test.mjs`, `tests/b.test.mjs`', [ ' - File: `tests/a.test.mjs`, `tests/b.test.mjs` (new)', ' - Verifies: x', ]); assert.deepEqual(testFirstFiles(body, '1.9'), ['tests/a.test.mjs', 'tests/b.test.mjs']); }); test('decision 20: a backticked path in the description of the File: line is not a test file, at 1.9 and below', () => { const body = step('`tests/a.test.mjs`', [ ' - File: `tests/a.test.mjs` (new), covers `tests/b.test.mjs`', ' - Verifies: x', ]); assert.deepEqual(testFirstFiles(body, '1.9'), ['tests/a.test.mjs']); // Operator decision 20 (12.10): below 1.9 too, only the paths right after File:. assert.deepEqual(testFirstFiles(body, '1.8'), ['tests/a.test.mjs']); }); test('decision 20: a code file named only in the File: line description keeps the step from being tests-only, at 1.9 and 1.8', () => { const files = '`tests/a.test.mjs`, `checks/smoke.mjs`'; const line = ' - File: `tests/a.test.mjs` (new), covers `checks/smoke.mjs`'; const body = step(files, [line, ' - Verifies: x']); assert.equal(m.stepTestsOnly(body, '1.9').testsOnly, false, JSON.stringify(m.stepTestsOnly(body, '1.9'))); assert.equal(m.stepTestsOnly(body, '1.8').testsOnly, false, JSON.stringify(m.stepTestsOnly(body, '1.8'))); const v19 = validatePlanContent(planWith('1.9', files, line)); assert.ok(!v19.warnings.some((w) => w.code === 'PLAN_STEP_TESTS_ONLY'), JSON.stringify(v19.warnings)); assert.deepEqual(v19.errors.filter((e) => PATH_CODES.includes(e.code)), [], JSON.stringify(v19.errors)); const v18 = validatePlanContent(planWith('1.8', files, line)); assert.ok(!v18.warnings.some((w) => w.code === 'PLAN_STEP_TESTS_ONLY'), JSON.stringify(v18.warnings)); }); test('1.9: a File: line that opens with `parse()` is refused as no path (error strict, warning soft)', () => { const text = planWith('1.9', '`tests/a.test.mjs`', ' - File: `parse()`'); const strict = validatePlanContent(text); const e = strict.errors.find((i) => i.code === 'PLAN_STEP_PATH_NOT_A_PATH'); assert.ok(e, JSON.stringify(strict.errors)); assert.match(e.message, /Step 1/); assert.match(e.message, /Test first/); const soft = validatePlanContent(text, { strict: false }); assert.ok(soft.warnings.some((w) => w.code === 'PLAN_STEP_PATH_NOT_A_PATH'), JSON.stringify(soft.warnings)); assert.deepEqual(soft.errors.filter((i) => PATH_CODES.includes(i.code)), []); }); test('1.9: an unticked File: line gets PLAN_STEP_PATH_NOT_FIRST next to PLAN_STEP_NO_TEST_FIRST', () => { const r = validatePlanContent(planWith('1.9', '`tests/a.test.mjs`', ' - File: tests/a.test.mjs')); const codes = r.errors.map((e) => e.code); assert.ok(codes.includes('PLAN_STEP_PATH_NOT_FIRST'), codes.join(',')); assert.ok(codes.includes('PLAN_STEP_NO_TEST_FIRST'), codes.join(',')); const e = r.errors.find((i) => i.code === 'PLAN_STEP_PATH_NOT_FIRST'); assert.match(e.message, /Step 1/); assert.match(e.message, /tests\/a\.test\.mjs/); }); // ---- operator decision 20 (12.10), review 6c31212c -------------------------- // Below 1.9 too, a step's test files are the backticked paths that stand right // after `File:`, separated only by commas or whitespace. A path in the // description after them (`covers parse() in src/parser.mjs`) is no test file, // so a step that builds that source is not tests-only and its source is never // locked. Each `File:` marker on a line starts its own run, as 6.3.1 read the // first span after each marker. // The review's exact input: the description names a call and the source. const REVIEW_FILES = '`src/parser.mjs`, `tests/parser.test.mjs`'; const REVIEW_LINE = ' - File: `tests/parser.test.mjs` (new) \u2014 covers `parse()` in `src/parser.mjs`'; test('decision 20: the review input (6c31212c) gives only the test below 1.9, is not tests-only, and gets no PLAN_STEP_TESTS_ONLY', () => { const body = step(REVIEW_FILES, [REVIEW_LINE, ' - Verifies: x']); for (const v of ['1.8', '1.7', null, undefined]) { assert.deepEqual(testFirstFiles(body, v), ['tests/parser.test.mjs'], `version ${v}`); } assert.equal(m.stepTestsOnly(body, '1.8').testsOnly, false, JSON.stringify(m.stepTestsOnly(body, '1.8'))); assert.equal(m.stepTestsOnly(body).testsOnly, false, JSON.stringify(m.stepTestsOnly(body))); const v18 = validatePlanContent(planWith('1.8', REVIEW_FILES, REVIEW_LINE)); assert.ok(!v18.warnings.some((w) => w.code === 'PLAN_STEP_TESTS_ONLY'), JSON.stringify(v18.warnings)); }); test('decision 20: two leading tests (comma or space) before a description that names a source give the two tests at 1.8 and 1.9', () => { for (const lead of ['`tests/a.test.mjs`, `tests/b.test.mjs`', '`tests/a.test.mjs` `tests/b.test.mjs`']) { const body = step('`tests/a.test.mjs`, `tests/b.test.mjs`, `src/a.mjs`', [ ` - File: ${lead} (new) - covers \`run()\` in \`src/a.mjs\``, ' - Verifies: x', ]); for (const v of ['1.8', '1.9']) { assert.deepEqual(testFirstFiles(body, v), ['tests/a.test.mjs', 'tests/b.test.mjs'], `${v}: ${lead}`); } } }); test('decision 20: a 1.8 line with two File: markers, each followed by a test, gives both tests (6.3.1 read both)', () => { const body = step('`tests/a.test.mjs`, `tests/b.test.mjs`', [ ' - File: `tests/a.test.mjs` (new) and File: `tests/b.test.mjs` (existing)', ' - Verifies: x', ]); assert.deepEqual(testFirstFiles(body, '1.8'), ['tests/a.test.mjs', 'tests/b.test.mjs']); assert.deepEqual(testFirstFiles(body, null), ['tests/a.test.mjs', 'tests/b.test.mjs']); }); // ---- operator decisions 28 (13.10) and 31 (14.10), review bd2520b0 --------- // Below 1.9 the run after `File:` also goes on past a note in brackets // (`(new)`, `*(new)*`, `(existing ...)`) and the whole words `and`, `plus` and // `+`, to a further path only when it is clearly a test; a path that is not // ends the run, and so does any other text. At 1.9 nothing changes. const OLDER = ['1.8', '1.7', null]; const fileLine = (rest) => step('`tests/a.test.mjs`, `tests/b.test.mjs`', [` - File: ${rest}`, ' - Verifies: x']); test('decisions 28 and 31: below 1.9 a test after a note, and, plus or + is read', () => { const lines = [ '`tests/a.test.mjs` (new), `tests/b.test.mjs` (new),', '`tests/a.test.mjs` and `tests/b.test.mjs`', '`tests/a.test.mjs` (existing) + `tests/b.test.mjs`', '`tests/a.test.mjs` (existing, the parser) plus `tests/b.test.mjs`', '`tests/a.test.mjs` *(new)*, `tests/b.test.mjs`', ]; for (const rest of lines) { for (const v of OLDER) { assert.deepEqual(testFirstFiles(fileLine(rest), v), ['tests/a.test.mjs', 'tests/b.test.mjs'], `version ${v}: ${rest}`); } } }); test('decisions 28 and 31: below 1.9 a near miss gives only the first test', () => { // Both read as tests by lib/util/test-path.mjs alone (a specs/ folder); the // run refuses them all the same: a document or a data file outside tests/. assert.equal(isTestPath('docs/specs/api.md'), true); assert.equal(isTestPath('api/specs/openapi.yaml'), true); const lines = [ '`tests/a.test.mjs` (new) - covers `src/a.mjs`', '`tests/a.test.mjs` (new), `src/a.mjs`', '`tests/a.test.mjs` (new) or `tests/b.test.mjs`', '`tests/a.test.mjs` (new), covers `tests/b.test.mjs`', '`tests/a.test.mjs` (new) and `src/a.mjs`', '`tests/a.test.mjs` (new) and `https://example.invalid/tests/b.test.mjs`', '`tests/a.test.mjs` (new) and `docs/specs/api.md`', '`tests/a.test.mjs` (new), `api/specs/openapi.yaml`', '`tests/a.test.mjs` android `tests/b.test.mjs`', '`tests/a.test.mjs` andand `tests/b.test.mjs`', '`tests/a.test.mjs` plusand `tests/b.test.mjs`', ]; for (const rest of lines) { for (const v of OLDER) { assert.deepEqual(testFirstFiles(fileLine(rest), v), ['tests/a.test.mjs'], `version ${v}: ${rest}`); } } }); test('decisions 28 and 31: below 1.9 the run after a note stops at the first source path', () => { const rest = '`tests/a.test.mjs` (new), `tests/b.test.mjs`, `src/a.mjs`, `tests/c.test.mjs`'; for (const v of OLDER) { assert.deepEqual(testFirstFiles(fileLine(rest), v), ['tests/a.test.mjs', 'tests/b.test.mjs'], `version ${v}`); } }); test('decisions 28 and 31: at 1.9 the two-test line with notes still gives only the first test', () => { const body = fileLine('`tests/a.test.mjs` (new), `tests/b.test.mjs` (new),'); assert.deepEqual(testFirstFiles(body, '1.9'), ['tests/a.test.mjs']); }); // ---- one definition in the prose (plan step 11, review cf241526) ----------- // The executor and the step helper define the step's test files once, as // decisions 20 and 31 read them; every other place points at that definition. // Operator decision 30 (14.10): red proof reads that list from the plan, so // neither the helper nor the executor passes --tests; it prints the list it // ran (`tests`), and the lock's --files is that printed list. A plan or step // red proof cannot read is `plan-unreadable`: the helper stops before it // builds, the executor corrects its own call and restarts the step once with // a fresh build helper (`build (restart)`), outside the fix rounds. A --tests // that differs is still refused (`tests-mismatch`, the plan's list in // `plan_tests`), but the documented call never gives one. const EXEC = ['commands', 'trekexecute.md']; const HELPER = ['agents', 'step-implementer.md']; const readRel = (rel) => readFileSync(join(ROOT, ...rel), 'utf8'); const count = (text, s) => text.split(s).length - 1; test('prose: neither file reads every backticked span on the File: line any more', () => { for (const rel of [EXEC, HELPER]) { const text = readRel(rel); assert.equal(count(text, 'every backticked span on that line'), 0, `${rel.join('/')} still says "every backticked span on that line"`); } }); test('prose: the decision-20 definition stands exactly once in each file and keeps the pinned phrase', () => { for (const rel of [EXEC, HELPER]) { const text = readRel(rel); assert.equal((text.match(/operator decision 20(?!\d)/g) || []).length, 1, `${rel.join('/')} holds the decision-20 definition once`); assert.ok(text.includes('every file on the `Test first:` `File:` line'), `${rel.join('/')} keeps the pinned phrase`); } }); test('prose: the executor definition names decisions 30 and 31, tests-mismatch, plan_tests and plan-unreadable, and keeps its line shape', () => { const lines = readRel(EXEC).split('\n'); const i = lines.findIndex((l) => l.startsWith("**The step's test files** are")); assert.ok(i >= 0, 'the definition line is there'); const def = lines[i]; assert.ok(def.includes('operator decision 20'), 'the definition line carries decision 20'); assert.ok(def.includes('operator decision 30'), 'the definition line carries decision 30'); assert.ok(def.includes('`tests-mismatch`'), 'the definition names tests-mismatch'); assert.ok(def.includes('`plan_tests`'), 'the definition names plan_tests'); assert.ok(def.includes('`plan-unreadable`'), 'the definition names plan-unreadable'); assert.ok(def.includes('`--files` is the list red proof printed as `tests`'), 'the definition says the lock follows the printed list'); assert.ok(def.endsWith(' The'), `the definition line still ends with " The": ${def.slice(-40)}`); assert.ok(lines[i + 1].startsWith('plan template lists them'), `the next line: ${lines[i + 1]}`); }); test('prose: both definitions name decision 31 and read past (new), and, plus or + below 1.9', () => { for (const rel of [EXEC, HELPER]) { const text = readRel(rel).replace(/\n\s*/g, ' '); const at20 = text.search(/operator decision 20(?!\d)/); const head = text.lastIndexOf('every file on the `Test first:` `File:` line', at20); assert.ok(at20 > 0 && head >= 0, `${rel.join('/')}: the definition is there`); const def = text.slice(head, at20); assert.ok(def.includes('operator decision 31'), `${rel.join('/')}: the definition names operator decision 31`); assert.ok(def.includes('`plus`'), `${rel.join('/')}: the definition names plus`); assert.ok(def.includes('`(new)`'), `${rel.join('/')}: the definition names (new)`); } }); test('prose: the executor exit-2 row names plan-unreadable first, the one restart outside the fix rounds, and tests-mismatch with plan_tests', () => { const row = readRel(EXEC).split('\n').find((l) => l.startsWith('| 2 | `"not-run"`')); assert.ok(row, 'the exit-2 row is there'); for (const s of ['`plan-unreadable`', '`tests-mismatch`', '`plan_tests`', 'fresh build helper', 'outside the fix rounds', 'absolute path', 'reruns only its own red proof', '`build (restart)`', '`cost_if_wrong`', '`attempts`', '`iterations_remaining`', '`error`', '`--resume`']) { assert.ok(row.includes(s), `the exit-2 row names ${s}`); } assert.match(row, /decision[- ]log\s+entry/, 'the exit-2 row records a decision-log entry'); }); test('prose: a second plan-unreadable goes up to the plan (stop class 4 row, Back to planning)', () => { const text = readRel(EXEC); const row4 = text.split('\n').find((l) => l.startsWith('| 4 | the plan |')); assert.ok(row4, 'the stop-class-4 row is there'); assert.ok(row4.includes('`plan-unreadable`'), 'the stop-class-4 row names plan-unreadable'); const b = text.indexOf('**Back to planning**'); assert.ok(b > 0, 'the Back to planning paragraph is there'); const end = text.indexOf('\n\n', b); const para = text.slice(b, end < 0 ? undefined : end); assert.ok(para.includes('`plan-unreadable`'), 'the Back to planning paragraph names plan-unreadable'); }); test('prose: the exit-1 otherwise row no longer counts tests_only: null', () => { const row = readRel(EXEC).split('\n').find((l) => l.startsWith('| 1, `not-red-at-base`, otherwise')); assert.ok(row, 'the exit-1 otherwise row is there'); assert.ok(!row.includes('tests_only: null'), 'the exit-1 otherwise row no longer says tests_only: null'); }); test('prose: no red-proof call line in either file passes --tests (operator decision 30)', () => { for (const rel of [EXEC, HELPER]) { const calls = readRel(rel).split('\n').filter((l) => l.includes('red-proof.mjs --red-only') || l.includes('red-proof.mjs --base')); assert.ok(calls.length > 0, `${rel.join('/')}: a red-proof call line is there`); for (const l of calls) assert.ok(!l.includes('--tests'), `${rel.join('/')}: the call passes --tests: ${l}`); } }); test('prose: below 1.8 a plan that names no test file gets --tests from the files the helper wrote (operator decision 41, review 7b3b59ed)', () => { const esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); const phrase = (s) => new RegExp(s.split(/\s+/).map(esc).join('\\s+')); const d41 = /\boperator\s+decision\s+41(?!\d)/; const NTFG = '`no test files given`'; const has = (text, s, where) => assert.match(text, phrase(s), `${where} holds "${s}"`); // The helper: the sentence that holds `no test files given`. const h = readRel(HELPER); const at = h.search(phrase(NTFG)); assert.ok(at >= 0, 'the helper names `no test files given`'); const before = h.slice(0, at); const ends = [...before.matchAll(/\.\s/g)]; const start = ends.length ? ends[ends.length - 1].index + 1 : 0; const rest = h.slice(at); const stop = rest.search(/\.(\s|$)/); const sentence = h.slice(start, stop < 0 ? undefined : at + stop + 1); has(sentence, 'below 1.8', 'the helper sentence'); has(sentence, '`--tests l.startsWith("**The step's test files** are")); assert.ok(def, 'the executor definition line is there'); for (const s of [NTFG, 'naming the test files it wrote', 'the `tests:` line the helper copied into its report']) { has(def, s, 'the executor definition line'); } assert.match(def, d41, 'the executor definition line names operator decision 41'); assert.ok(def.endsWith(' The'), `the definition line still ends with " The": ${def.slice(-40)}`); const row = lines.find((l) => l.startsWith('| 2 | `"not-run"`')); assert.ok(row, 'the exit-2 row is there'); const r = row.search(phrase(NTFG)); assert.ok(r >= 0, 'the exit-2 row names `no test files given`'); const after = row.slice(r); for (const s of ['is not a finding', 'the `tests:` line in the helper\'s report', 'tolerated as `"not-run"`']) { has(after, s, 'the exit-2 row after `no test files given`'); } assert.match(after, d41, 'the exit-2 row names operator decision 41'); // Handover 4, invariant 7. const ho = readRel(['docs', 'HANDOVER-CONTRACTS.md']); const a7 = ho.indexOf('7. **Every plan_version'); const a8 = ho.indexOf('8. **', a7); assert.ok(a7 >= 0 && a8 > a7, 'Handover 4 invariants 7 and 8 are there'); const inv7 = ho.slice(a7, a8); for (const s of [NTFG, 'naming the test files it wrote', 'the `tests:` line in the helper\'s report']) { has(inv7, s, 'Handover 4 invariant 7'); } assert.match(inv7, d41, 'Handover 4 invariant 7 names operator decision 41'); }); test('prose: the helper stops on plan-unreadable before the 1.8 branches and locks the printed list', () => { const text = readRel(HELPER); const pu = text.match(/plan-unreadable[^.]*do not build/); assert.ok(pu, 'the helper does not build on plan-unreadable'); const below = text.indexOf('Below 1.8'); assert.ok(below > 0, 'the helper still has its Below 1.8 branch'); assert.ok(pu.index < below, 'the plan-unreadable sentence stands before "Below 1.8"'); assert.ok(text.includes('`--files` is the list red proof printed'), 'the lock follows the printed list'); assert.ok(text.includes('copy that line into the report'), 'the helper copies the tests line into the report'); }); test('prose: the round label build (restart) is named by the executor prompt, the report heading and the helper build mode', () => { const ex = readRel(EXEC); assert.ok(ex.split('\n').some((l) => l.includes('(`build`, `build (restart)`, or `fix round R`)')), 'the Sub-step C prompt line names build (restart)'); const h = readRel(HELPER); assert.ok(h.split('\n').includes('## Step {N} - {build | build (restart) | fix round R}'), 'the helper report heading names build (restart)'); const a = h.indexOf('## Mode `build`'); const z = h.indexOf('## Mode `fix`'); assert.ok(a >= 0 && z > a, 'the helper has both modes'); assert.match(h.slice(a, z), /build \(restart\)[^.]*one\s+copy\s+of\s+each\s+test\s+case/); }); test('prose: the old rerun wording is gone from both files', () => { const ex = readRel(EXEC); for (const s of ['runs it once more with the list it printed', 'reruns once with the list red proof printed', 'the list red proof accepted']) { assert.ok(!ex.includes(s), `commands/trekexecute.md still says "${s}"`); } const h = readRel(HELPER); assert.doesNotMatch(h, /once\s+more[^.]*plan_tests/); for (const s of ['after a rerun, `plan_tests`', 'the list red proof accepted']) { assert.ok(!h.includes(s), `agents/step-implementer.md still says "${s}"`); } });