Kjell Tore Guttormsen
a10018f9d3
Round 8 (0f400d3..29b4ac9, BLOCK: 1 BLOCKER, 12 MAJOR, 1 MINOR). Each fix
began with a test that failed on 29b4ac9 (13 tests, all red there); suite
2221 (2219/0/2).
- BLOCKER: a backticked check whose trailing comment holds a quote, $,
backtick or backslash is refused, not dropped. The comment was never
screened, while the executor hands the Verify span as written to red
proof, the test lock and the done line inside --cmd "...", where a $(...)
runs and a quote ends the argument. trekplan's text and the reviewer's
NOT RUN table say so.
- A span that is only a comment is not-a-command, not a placeholder. The
dropComment text says the allowlist still refuses a word with a # in it.
- The criteria runner, comparison-gate and end-state-gate use isMain: they
did nothing and exited 0 under a symlinked path. The test now refuses any
process.argv[1] comparison outside lib/util/is-main.mjs.
- Files: in a field with backticked paths, a plain word counts only before
the entry's description (a dash, or a colon after the path); a name with
no slash only first in its entry or after and/or/+/&. "mocks
fs.readFileSync", "runs on Node.js 22.x" or "no change to package.json"
no longer make a tests-only step look like it builds something.
Lowercase makefile, rakefile and justfile are build files. Measured on the
520 Files: fields in 48 plans of the marketplace and the test repo:
3 fields read differently (two notes no longer read as files, one
duplicate name dropped), tests-only changed in 0.
- The test lock compares --cmd without its trailing comment, and the test
lock and the done line read a stop with stoppedBy, the one reading.
red-proof drops its second reading, spawnError.
- release-check reads a relative --review in --repo or, when only that one
exists, from the current directory as 6.3.0 did; two different files are
exit 2, never guessed. The runbook says so.
- One trailing-json reader: a closing fence may be longer than three
backticks, in the block reader and the verdict gate alike, and the review
validator and the synthesis digest use it, so three backticks inside a
JSON string no longer hide a BLOCKER from the remediation-step check.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1026 lines
66 KiB
JavaScript
1026 lines
66 KiB
JavaScript
// tests/lib/review-630.test.mjs
|
|
// The findings of the release review of 6.3.0 (2026-10-10), each held by a
|
|
// test that failed on f669b87. The guard's and the release check's own tests
|
|
// live next to their files (tests/commands/trekexecute-coupling.test.mjs,
|
|
// tests/hooks/guards-review-regressions.test.mjs, tests/scripts/release-check.test.mjs).
|
|
|
|
import { test, mock } from 'node:test';
|
|
import { strict as assert } from 'node:assert';
|
|
import { spawnSync } from 'node:child_process';
|
|
import fs from 'node:fs';
|
|
import { cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
|
|
import { syncBuiltinESMExports } from 'node:module';
|
|
import { tmpdir } from 'node:os';
|
|
import { dirname, join } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { buildPlan } from '../helpers/plan-v18-builder.mjs';
|
|
import { validatePlanContent } from '../../lib/validators/plan-validator.mjs';
|
|
import { stepTestsOnly } from '../../lib/parsers/plan-schema.mjs';
|
|
import { validateFindings } from '../../lib/review/findings-schema.mjs';
|
|
import { lintSkill } from '../../lib/skills/skill-lint.mjs';
|
|
import * as routing from '../../lib/skills/routing.mjs';
|
|
|
|
const awaitImportSync = () => routing;
|
|
|
|
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
|
const read = (rel) => readFileSync(join(ROOT, rel), 'utf8');
|
|
const VALIDATOR = join(ROOT, 'lib', 'validators', 'plan-validator.mjs');
|
|
|
|
// The review template's own Step R1, appended the Phase 7.2 way.
|
|
function withTemplateStep(plan, n) {
|
|
const r1 = /### Step R1:[^\n]*\n[\s\S]*?(?=\n> )/.exec(read('templates/trekreview-template.md'));
|
|
assert.ok(r1, 'templates/trekreview-template.md carries a ### Step R1');
|
|
// Phase 7.2 step 2 also adds the manifest the legacy rule synthesizes (expected_paths ← Files).
|
|
const appended = r1[0].replace(/^### Step R1:/, `### Step ${n}:`) +
|
|
'- **Checkpoint:** `git commit -m "fix(review): x [01234567]"`\n' +
|
|
'- **Manifest:**\n ```yaml\n manifest:\n expected_paths:\n - lib/foo.mjs\n min_file_count: 1\n' +
|
|
' commit_message_pattern: "^fix\\\\(review\\\\):"\n bash_syntax_check: []\n forbidden_paths: []\n must_contain: []\n ```\n';
|
|
return plan.replace('\n## Test Strategy', `\n${appended}\n## Test Strategy`);
|
|
}
|
|
|
|
// ---- BLOCKER 1b25ba56: appended review steps hold the 1.8 step rules on any plan
|
|
|
|
test('review 630 / 1b25ba56: on a 1.7 plan, an appended review step that cannot run is refused (--steps-from)', () => {
|
|
const dir = mkdtempSync(join(tmpdir(), 'r630-'));
|
|
try {
|
|
const plan = withTemplateStep(buildPlan({ version: '1.7', design: false }), 3);
|
|
const p = join(dir, 'plan.md');
|
|
writeFileSync(p, plan);
|
|
const plain = spawnSync(process.execPath, [VALIDATOR, '--strict', '--json', p], { encoding: 'utf8' });
|
|
assert.equal(plain.status, 0, 'the control: a 1.7 plan is not held to the 1.8 step rules');
|
|
const r = spawnSync(process.execPath, [VALIDATOR, '--strict', '--json', '--steps-from', '3', p], { encoding: 'utf8' });
|
|
assert.equal(r.status, 1, `the appended step is held to them: ${r.stdout}`);
|
|
const codes = JSON.parse(r.stdout).errors.map((e) => e.code);
|
|
assert.ok(codes.includes('PLAN_STEP_VERIFY_NOT_RUNNABLE') || codes.includes('PLAN_STEP_NO_TEST_FIRST'), codes.join(','));
|
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('review 630 / 1b25ba56: --steps-from leaves the earlier steps of a 1.7 plan alone', () => {
|
|
const plan = buildPlan({ version: '1.7', design: false, steps: [{ testFirst: false }, { endToEnd: 'bash tests/e2e.sh' }] });
|
|
assert.deepEqual(validatePlanContent(plan, { strict: true, stepsFrom: 3 }).errors, [],
|
|
'step 1 has no Test first, and it is not one of the appended steps');
|
|
});
|
|
|
|
test('review 630 / 1b25ba56: Phase 7.2 validates the appended steps with --steps-from', () => {
|
|
const cmd = read('commands/trekexecute.md');
|
|
assert.match(cmd, /plan-validator\.mjs --strict --json --steps-from \{total \+ 1\} "\{plan_path\}"/);
|
|
});
|
|
|
|
// ---- 57a5e67c: the wave path names the guard's refusal ------------------------
|
|
|
|
test('review 630 / 57a5e67c: the wave merge path says what to do when the guard refused because no one can be asked', () => {
|
|
const cmd = read('commands/trekexecute.md');
|
|
const at = cmd.indexOf('**Merge session branches back (SEQUENTIAL, one at a time):**');
|
|
const before = cmd.slice(cmd.lastIndexOf('- **`all_completed: true`:**', at), at);
|
|
assert.match(before, /no one to ask/, 'the guard-refused case is named on the wave path');
|
|
assert.match(before, /by hand/, 'and the way on: the operator runs the merge by hand');
|
|
assert.match(before, /wave\.json/, 'wave.json is kept');
|
|
});
|
|
|
|
// ---- 2cf29d6f / 727924c0: what counts as a step of only tests ------------------
|
|
|
|
test('review 630: a comma inside a Files: note is not a path', () => {
|
|
const body = '- **Files:** `tests/a.test.mjs` (new: covers x, y)\n- **Changes:** add the test\n';
|
|
assert.deepEqual(stepTestsOnly(body), { testsOnly: true, files: ['tests/a.test.mjs'] });
|
|
});
|
|
|
|
test('review 630: a data file under specs/ is not a test', () => {
|
|
const body = '- **Files:** `api/specs/openapi.yaml`\n- **Changes:** add the endpoint\n';
|
|
assert.equal(stepTestsOnly(body).testsOnly, false);
|
|
const fixture = '- **Files:** `tests/fixtures/users.json`\n- **Changes:** add the fixture\n';
|
|
assert.equal(stepTestsOnly(fixture).testsOnly, true, 'the control: a fixture under tests/ is a test file');
|
|
});
|
|
|
|
test('review 630: a tests-only End-to-end step is told where End-to-end belongs', () => {
|
|
const plan = buildPlan({ steps: [{}, { file: 'tests/e2e.sh', test: 'tests/e2e.sh', endToEnd: 'bash tests/e2e.sh' }] });
|
|
const w = validatePlanContent(plan, { strict: true }).warnings.find((x) => x.code === 'PLAN_STEP_TESTS_ONLY');
|
|
assert.ok(w, 'the warning fires: red proof would reject this step');
|
|
assert.match(`${w.message} ${w.hint ?? ''}`, /`End-to-end:` (rides|belongs) on the last step that builds/);
|
|
assert.match(read('templates/plan-template.md'), /`End-to-end:` (rides|belongs) on the last step that builds/);
|
|
});
|
|
|
|
// ---- c4623362: a removed line can still be cited --------------------------------
|
|
|
|
test('review 630 / c4623362: the out-of-range hint says how to cite a line that no longer exists', () => {
|
|
const r = validateFindings({ reviewer: 'r', findings: [{
|
|
id: 'a'.repeat(40), severity: 'MAJOR', rule_key: 'MISSING_TEST', file: 'package.json', line: 1,
|
|
brief_ref: 'x', title: 't', recommended_action: 'a',
|
|
detail: 'the check formerly at package.json:9999 was removed',
|
|
}] }, { repoRoot: ROOT });
|
|
const e = r.errors.find((x) => x.code === 'FINDING_PROSE_LINE_OUT_OF_RANGE');
|
|
assert.ok(e, 'the control: the plain citation is still checked');
|
|
assert.match(e.hint, /<rev>:<path>:<line>/);
|
|
const ok = validateFindings({ reviewer: 'r', findings: [{
|
|
id: 'a'.repeat(40), severity: 'MAJOR', rule_key: 'MISSING_TEST', file: 'package.json', line: 1,
|
|
brief_ref: 'x', title: 't', recommended_action: 'a',
|
|
detail: 'the check formerly at v6.2.0:package.json:9999 was removed',
|
|
}] }, { repoRoot: ROOT });
|
|
assert.deepEqual(ok.errors, [], 'a revision-qualified citation is not judged against today\'s file');
|
|
assert.match(read('commands/trekreview.md'), /<rev>:<path>:<line>/, 'the re-ask names the form');
|
|
});
|
|
|
|
// ---- 2261016d: a 499-line SKILL.md is under the 500-line limit -------------------
|
|
|
|
test('review 630 / 2261016d: a SKILL.md of 499 lines ending in a newline passes the size rule', () => {
|
|
const dir = mkdtempSync(join(tmpdir(), 'r630-skill-'));
|
|
try {
|
|
const skill = join(dir, 'test-first');
|
|
cpSync(join(ROOT, 'skills', 'test-first'), skill, { recursive: true });
|
|
const text = readFileSync(join(skill, 'SKILL.md'), 'utf8').replace(/\n+$/, '\n');
|
|
const lines = text.split('\n').length - 1;
|
|
assert.ok(lines < 499, `the fixture pads the real skill (${lines} lines)`);
|
|
const padded = text.replace('\n## Credits', `${'\n'.repeat(499 - lines)}\n## Credits`);
|
|
assert.equal(padded.split('\n').length - 1, 499);
|
|
writeFileSync(join(skill, 'SKILL.md'), padded);
|
|
const codes = lintSkill(skill).errors.map((e) => e.code);
|
|
assert.ok(!codes.includes('SKILL_TOO_LONG'), codes.join(','));
|
|
writeFileSync(join(skill, 'SKILL.md'), padded + '\n');
|
|
assert.ok(lintSkill(skill).errors.map((e) => e.code).includes('SKILL_TOO_LONG'), 'the control: 500 lines fail');
|
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
|
});
|
|
|
|
// ---- cb34c32e: regression-check takes commandEnv from the shared util ------------
|
|
|
|
test('review 630 / cb34c32e: regression-check takes commandEnv from lib/util/command-env.mjs', () => {
|
|
// Round 5 (MINOR 0e98d5c7): regression-check has no exec of its own any more;
|
|
// it runs through the criteria runner's runCommand, which takes commandEnv
|
|
// from lib/util/command-env.mjs.
|
|
const src = read('lib/verification/regression-check.mjs');
|
|
assert.doesNotMatch(src, /from '\.\/red-proof\.mjs'/);
|
|
assert.match(src, /runCriteria\(c, o\)/);
|
|
assert.match(read('lib/verification/criteria-runner.mjs'), /import \{ commandEnv \} from '\.\.\/util\/command-env\.mjs';/);
|
|
});
|
|
|
|
// ---- CLAUDE.md names the mode-read exemptions as they are -----------------------
|
|
|
|
test('review 630: CLAUDE.md counts the mode-read exemptions the launch test holds', () => {
|
|
const src = read('tests/lib/no-claude-code-launch.test.mjs');
|
|
const n = (src.match(/\['mode-read', \d+,/g) || []).length;
|
|
assert.equal(n, 4, 'the launch test holds four mode-read entries');
|
|
assert.match(read('CLAUDE.md'), /four files that only read\s+(?:> )?the session's permission mode/);
|
|
});
|
|
|
|
// ---- /security-review, older than v6.2.0: a read-only git that writes a file ----
|
|
|
|
test('review 630 / security: `git diff --output=<path>` is not a read-only check', async () => {
|
|
const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
|
assert.equal(allowedCommand('git diff --stat').allowed, true, 'the control: a read-only git check runs');
|
|
assert.equal(allowedCommand('git log --output-indicator-new=+ -1').allowed, true, 'an option that only starts with --output is not it');
|
|
for (const cmd of ['git diff --output=../../.zshrc', 'git diff --output ../x', 'git log -p --output=x', 'git show HEAD --output=x']) {
|
|
const r = allowedCommand(cmd);
|
|
assert.equal(r.allowed, false, `${cmd} writes a file`);
|
|
assert.match(r.reason, /--output/);
|
|
}
|
|
});
|
|
|
|
// ==== round 2 of the release review (2026-10-10), each red on 0d044f6 ===========
|
|
|
|
test('review 630 r2 / BLOCKER: a quoted, escaped or expanded --output is refused too', async () => {
|
|
const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
|
for (const cmd of [
|
|
'git diff "--output=/tmp/x"', "git log -p '--output'=x", "git diff '--output=q1'", 'git log "--output" q1',
|
|
'git diff --output\\=x', 'git diff --\\output=q1', 'git diff --out{put,x}=y', 'git diff --outp$EMPTY=x',
|
|
'git diff --outp?t=x', 'git diff --outpu=x', 'git log --out x',
|
|
]) {
|
|
assert.equal(allowedCommand(cmd).allowed, false, `${cmd} must not reach the shell`);
|
|
}
|
|
for (const cmd of ['git log --oneline -1', 'git diff --stat HEAD~1', 'git show HEAD^:package.json', 'git log --format=%h -1',
|
|
'git ls-files tests/', 'git log --output-indicator-new=+ -1']) {
|
|
assert.equal(allowedCommand(cmd).allowed, true, `the control: ${cmd} is a read-only check`);
|
|
}
|
|
});
|
|
|
|
test('review 630 r2: a Files: entry without backticks names every path in it', () => {
|
|
const body = '- **Files:** tests/a.test.mjs + src/b.mjs\n- **Changes:** x\n';
|
|
assert.deepEqual(stepTestsOnly(body), { testsOnly: false, files: ['tests/a.test.mjs', 'src/b.mjs'] });
|
|
const and = '- **Files:** tests/a.test.mjs and src/b.mjs\n- **Changes:** x\n';
|
|
assert.equal(stepTestsOnly(and).testsOnly, false);
|
|
const plain = '- **Files:** Makefile\n- **Changes:** x\n';
|
|
assert.deepEqual(stepTestsOnly(plain).files, ['Makefile'], 'the control: a bare file name is still a path');
|
|
});
|
|
|
|
test('review 630 r2: a backticked path with parentheses keeps them', () => {
|
|
const body = '- **Files:** `app/(auth)/login.check.sh` (new)\n- **Changes:** x\n' +
|
|
'- **Test first:**\n - File: `app/(auth)/login.check.sh` (new)\n - Verifies: y\n';
|
|
assert.deepEqual(stepTestsOnly(body), { testsOnly: true, files: ['app/(auth)/login.check.sh'] });
|
|
});
|
|
|
|
test('review 630 r2: --steps-from reads the plan\'s ## Verification for an appended End-to-end step', () => {
|
|
const plan = buildPlan({ version: '1.7', design: false });
|
|
const e2e = '### Step 3: Run it\n\n- **Files:** `src/run.mjs`\n- **Changes:** wire it\n' +
|
|
'- **Test first:**\n - File: `tests/run.test.mjs` (new)\n - Verifies: it runs\n' +
|
|
'- **Verify:** `node --test tests/run.test.mjs` → expected: exit 0\n' +
|
|
'- **End-to-end:** `bash tests/e2e.sh` → expected: the CLI prints ok\n' +
|
|
'- **Manifest:**\n ```yaml\n manifest:\n expected_paths:\n - src/run.mjs\n min_file_count: 1\n' +
|
|
' commit_message_pattern: "^fix"\n bash_syntax_check: []\n forbidden_paths: []\n must_contain: []\n ```\n';
|
|
const edited = plan.replace('\n## Test Strategy', `\n${e2e}\n## Test Strategy`);
|
|
assert.match(edited, /- \[ \] `bash tests\/e2e\.sh`/, 'the fixture lists the command under ## Verification');
|
|
const r = validatePlanContent(edited, { strict: true, stepsFrom: 3 });
|
|
assert.deepEqual(r.errors.map((e) => e.code), [], JSON.stringify(r.errors));
|
|
});
|
|
|
|
test('review 630 r2: --soft --steps-from warns instead of failing', () => {
|
|
const plan = withTemplateStep(buildPlan({ version: '1.7', design: false }), 3);
|
|
const soft = validatePlanContent(plan, { strict: false, stepsFrom: 3 });
|
|
assert.equal(soft.valid, true, JSON.stringify(soft.errors));
|
|
assert.ok(soft.warnings.some((w) => /VERIFY_NOT_RUNNABLE|TEST_FIRST/.test(w.code)), 'the issue is still reported');
|
|
assert.equal(validatePlanContent(plan, { strict: true, stepsFrom: 3 }).valid, false, 'the control: strict fails');
|
|
});
|
|
|
|
test('review 630 r2: regression-check prints the status progress.json takes', async () => {
|
|
const { checkRegression } = await import('../../lib/verification/regression-check.mjs');
|
|
const { PROGRESS_REGRESSION_VALUES } = await import('../../lib/validators/progress-validator.mjs');
|
|
const plan = '## Implementation Plan\n\n### Step 1: x\n\n- **Regression:** `npm test`\n';
|
|
const blocked = checkRegression({ planText: plan, step: 1, run: () => [{ status: 'blocked', command: 'npm test' }] });
|
|
assert.equal(blocked.progress_status, 'unrunnable');
|
|
const missing = checkRegression({ planText: plan, step: 9 });
|
|
assert.equal(missing.progress_status, 'not-run');
|
|
for (const r of [blocked, missing]) assert.ok(PROGRESS_REGRESSION_VALUES.includes(r.progress_status));
|
|
assert.match(read('commands/trekexecute.md'), /`progress_status`/, 'the executor writes that field');
|
|
});
|
|
|
|
test('review 630 r2: a skill is found by its name even when an agent has the same name', () => {
|
|
const dir = mkdtempSync(join(tmpdir(), 'r630-route-'));
|
|
try {
|
|
for (const d of ['skills', 'commands', 'agents', 'evals']) cpSync(join(ROOT, d), join(dir, d), { recursive: true });
|
|
writeFileSync(join(dir, 'agents', 'debugging.md'),
|
|
'---\nname: debugging\ndescription: Bakes sourdough loaves and schedules oven time for the shared kitchen rota.\nmodel: opus\n---\n\nBread.\n');
|
|
const { checkRouting } = awaitImportSync();
|
|
const codes = checkRouting(dir).errors.map((e) => e.code);
|
|
assert.ok(!codes.includes('ROUTING_UNKNOWN_SKILL'), codes.join(','));
|
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('review 630 r2: rank() does not rebuild the document vectors per call', () => {
|
|
const { buildCorpus, rank } = awaitImportSync();
|
|
const corpus = buildCorpus([
|
|
{ id: 'a', name: 'alpha', description: 'parses plans and validates steps' },
|
|
{ id: 'b', name: 'beta', description: 'runs tests and reports failures' },
|
|
]);
|
|
let calls = 0;
|
|
const idf = corpus.idf;
|
|
corpus.idf = (t) => { calls++; return idf(t); };
|
|
rank('validates plans', corpus);
|
|
rank('reports failures', corpus);
|
|
assert.ok(calls <= 4, `idf was called ${calls} times for two two-word prompts`);
|
|
});
|
|
|
|
test('review 630 r2 (operator 2026-10-10): Phase 7.2 says plainly that template-form steps stop with a re-plan', () => {
|
|
const cmd = read('commands/trekexecute.md');
|
|
assert.match(cmd, /Today the loop runs only for steps written as\s+runnable plan steps/);
|
|
assert.match(cmd, /re-plan from the review" — the\s+usual outcome until a later order/);
|
|
});
|
|
|
|
// ==== round 3 of the release review (2026-10-10), each red on 96cb977 ===========
|
|
|
|
test('review 630 r3 / BLOCKER: no runner takes an option that writes a file or loads code', async () => {
|
|
const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
|
for (const cmd of [
|
|
'node --test --test-reporter=tap --test-reporter-destination=/tmp/x',
|
|
'node --test --import=data:text/javascript,1',
|
|
'node --test "--import=data:text/javascript,import(\'node:fs\')"',
|
|
'node --test --require ./evil.cjs',
|
|
'node --test --test-reporter=./evil-reporter.mjs',
|
|
'node --test --test-reporter ./evil-reporter.mjs',
|
|
'jest --json --outputFile=/tmp/x',
|
|
'jest --config=evil.js',
|
|
'vitest --outputFile=/tmp/x',
|
|
'vitest run --reporter=./evil.mjs',
|
|
'pytest --junitxml=/tmp/x',
|
|
'pytest --basetemp=/tmp/anydir',
|
|
'pytest -p evil_plugin',
|
|
'python -m pytest --junitxml=/tmp/x',
|
|
'uv run pytest --basetemp=/tmp/x',
|
|
'go test -o /tmp/x',
|
|
'go test -exec /tmp/evil ./...',
|
|
'cargo test --target-dir=/tmp/x',
|
|
'npm test -- --test-reporter-destination=/tmp/x',
|
|
'npm test --script-shell=/tmp/evil',
|
|
'npm run verify -- --outputFile=/tmp/x',
|
|
'make test SHELL=/tmp/evil',
|
|
'make test -f /tmp/evil.mk',
|
|
'bash tests/x.sh "$(id)"',
|
|
]) {
|
|
assert.equal(allowedCommand(cmd, { cwd: ROOT }).allowed, false, `${cmd} must not reach the shell`);
|
|
}
|
|
for (const cmd of [
|
|
'npm test', 'npm test -- tests/lib/criteria-runner.test.mjs', 'node --test tests/lib/criteria-runner.test.mjs',
|
|
'node --test tests/lib/*.test.mjs', 'node --test --test-reporter=tap tests/lib/review-630.test.mjs', 'node --test --test-reporter dot', 'node --test --test-name-pattern=review tests/lib/review-630.test.mjs',
|
|
'vitest run', 'jest --ci', 'pytest -q', 'pytest -x -q tests/', 'python3 -m pytest tests/', 'uv run pytest',
|
|
'cargo test', 'cargo test --release', 'go test ./...', 'go test -run TestX -count=1 ./pkg', 'make test',
|
|
'bash tests/fixtures/criteria-exit-0.sh', 'bash tests/fixtures/criteria-exit-0.sh --fast',
|
|
]) {
|
|
const r = allowedCommand(cmd, { cwd: ROOT });
|
|
assert.equal(r.allowed, true, `the control: ${cmd} is a test run (${r.reason})`);
|
|
}
|
|
});
|
|
|
|
test('review 630 r3: a restore the guard refuses is never recorded as reverted', async () => {
|
|
const { PROGRESS_SIMPLIFY_VALUES } = await import('../../lib/validators/progress-validator.mjs');
|
|
assert.ok(PROGRESS_SIMPLIFY_VALUES.includes('revert-refused'), 'progress.json can say the restore was refused');
|
|
assert.match(read('docs/HANDOVER-CONTRACTS.md'), /`applied \\| clean \\| reverted \\| revert-refused \\| not-run`/);
|
|
const cmd = read('commands/trekexecute.md');
|
|
const onFailure = cmd.slice(cmd.indexOf('**`On failure: revert`** and **`On failure: retry`**'), cmd.indexOf('**`On failure: skip`**'));
|
|
assert.match(onFailure, /guard refuses the restore/, 'the step revert says what happens when the guard refuses it');
|
|
const simplify = cmd.slice(cmd.indexOf('2. **`/simplify`**'), cmd.indexOf('3. **`/trekreview`**'));
|
|
assert.match(simplify, /simplify = "revert-refused"/);
|
|
assert.match(simplify, /never `"reverted"`/);
|
|
});
|
|
|
|
test('review 630 r3: a fixture document under tests/ is a test file, and "e.g." is not a path', () => {
|
|
const step = (files) => `- **Files:** ${files}\n- **Changes:** x\n`;
|
|
assert.equal(stepTestsOnly(step('`tests/a.test.mjs`, `tests/fixtures/sample.md`')).testsOnly, true);
|
|
assert.deepEqual(stepTestsOnly(step('tests/a.test.mjs (new), e.g. the fixture tests/fixtures/x.json')),
|
|
{ testsOnly: true, files: ['tests/a.test.mjs', 'tests/fixtures/x.json'] });
|
|
assert.equal(stepTestsOnly(step('`docs/specs/api.md`')).testsOnly, false, 'a document outside tests/ is still not a test');
|
|
assert.deepEqual(stepTestsOnly(step('`./tests/a.test.mjs`')).files, ['tests/a.test.mjs'], './ is stripped');
|
|
});
|
|
|
|
test('review 630 r3: the plan-file sweep and the tests-only check both drop a leading ./', async () => {
|
|
// Round 3 had the sweep read Files: through the tests-only reader; the
|
|
// operator reversed that on 2026-10-11 (the K6 sweep keeps v6.2.0's reader,
|
|
// round 5). What the round-3 finding named still holds: `./` is dropped by both.
|
|
const { plannedPaths } = await import('../../lib/verification/plan-scope-sweep.mjs');
|
|
const plan = '## Implementation Plan\n\n### Step 1: x\n\n- **Files:** `./src/a.mjs`, `./tests/b.test.mjs`\n- **Changes:** x\n';
|
|
const body = plan.slice(plan.indexOf('### Step 1'));
|
|
assert.deepEqual(plannedPaths(plan), ['src/a.mjs', 'tests/b.test.mjs']);
|
|
assert.deepEqual(stepTestsOnly(body).files, ['src/a.mjs', 'tests/b.test.mjs']);
|
|
});
|
|
|
|
test('review 630 r3: --steps-from refuses an appended step with no Verify: on a 1.8 plan too', () => {
|
|
const plan = buildPlan({});
|
|
const step = '### Step 3: Fix it\n\n- **Files:** `src/fix.mjs`, `tests/fix.test.mjs`\n- **Changes:** fix it\n' +
|
|
'- **Test first:**\n - File: `tests/fix.test.mjs` (new)\n - Verifies: it is fixed\n- **On failure:** escalate\n';
|
|
const edited = plan.replace('\n## Test Strategy', `\n${step}\n## Test Strategy`);
|
|
assert.ok(edited.includes('### Step 3: Fix it'), 'the fixture appends the step');
|
|
const r = validatePlanContent(edited, { strict: true, stepsFrom: 3 });
|
|
assert.ok(r.errors.some((e) => e.code === 'PLAN_STEP_VERIFY_NOT_RUNNABLE' && /Step 3/.test(e.message)),
|
|
JSON.stringify(r.errors.map((e) => e.code)));
|
|
});
|
|
|
|
test('review 630 r3: the plan validator hands its parsed fields to the tests-only check', async () => {
|
|
const schema = await import('../../lib/parsers/plan-schema.mjs');
|
|
assert.equal(typeof schema.stepTestsOnlyFromFields, 'function');
|
|
const body = '- **Files:** `tests/a.test.mjs`\n- **Changes:** x\n';
|
|
assert.deepEqual(schema.stepTestsOnlyFromFields(schema.parseStepFields(body)), stepTestsOnly(body));
|
|
assert.match(read('lib/validators/plan-validator.mjs'), /stepTestsOnlyFromFields\(f\)/);
|
|
});
|
|
|
|
test('review 630 r3: skill frontmatter - a value starting with # is a hazard, an escaped backslash closes a quote', async () => {
|
|
const { plainScalarHazards, readFrontmatterFields } = await import('../../lib/skills/skill-frontmatter.mjs');
|
|
assert.ok(plainScalarHazards('#1 helper. Use when testing.').length > 0, 'YAML reads "#..." as a comment: the value is null');
|
|
const r = readFrontmatterFields('---\nname: x\ndescription: "Path C:\\\\"\n---\nbody\n');
|
|
assert.equal(r.fields.description?.value, 'Path C:\\', JSON.stringify(r));
|
|
const open = readFrontmatterFields('---\nname: x\ndescription: "a \\"\n---\nbody\n');
|
|
assert.ok(JSON.stringify(open).includes('unclosed double quote'), 'a quote escaped by one backslash is still open');
|
|
});
|
|
|
|
test('review 630 r3: routing reads a kind tag only from its own table, never Object.prototype', () => {
|
|
const dir = mkdtempSync(join(tmpdir(), 'r630-proto-'));
|
|
try {
|
|
for (const d of ['skills', 'commands', 'agents', 'evals']) cpSync(join(ROOT, d), join(dir, d), { recursive: true });
|
|
const c = join(dir, 'evals', 'zz-proto');
|
|
cpSync(join(ROOT, 'evals', 'skill-test-first-quiet-1'), c, { recursive: true });
|
|
writeFileSync(join(c, 'prompt.md'), '---\ntags: [constructor, owner=nobody]\n---\n\nWhat time is it?\n');
|
|
const { checkRouting } = awaitImportSync();
|
|
const r = checkRouting(dir);
|
|
assert.ok(!r.errors.some((e) => String(e.message ?? '').includes('zz-proto') || e.case === 'zz-proto'), JSON.stringify(r.errors));
|
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
|
});
|
|
|
|
// ==== round 4 of the release review (2026-10-10), each red on 41ea8fc ===========
|
|
|
|
test('review 630 r4: the Files: reader keeps every path the field names, and only those', async () => {
|
|
const { fieldPaths } = await import('../../lib/parsers/plan-schema.mjs');
|
|
const cases = [
|
|
['`src/a.mjs` (fixture: `tests/fixtures/a.json`)', ['src/a.mjs', 'tests/fixtures/a.json']],
|
|
['`tests/a.test.mjs`, `src/a.mjs`.', ['tests/a.test.mjs', 'src/a.mjs']],
|
|
['`src/cli.mjs`: add the flag; `src/b.mjs`.', ['src/cli.mjs', 'src/b.mjs']],
|
|
['`tests/a.test.mjs` — covers parse, format', ['tests/a.test.mjs']],
|
|
['docs/ and README.md', ['docs/', 'README.md']],
|
|
['app/(auth)/x.ts', ['app/(auth)/x.ts']],
|
|
['src/*.mjs (all of them)', ['src/*.mjs']],
|
|
['notes for v6.3.0, src/x.mjs', ['src/x.mjs']],
|
|
['Makefile', ['Makefile']],
|
|
['tests/a.test.mjs + src/b.mjs', ['tests/a.test.mjs', 'src/b.mjs']],
|
|
];
|
|
for (const [text, want] of cases) assert.deepEqual(fieldPaths(text), want, text);
|
|
const body = '- **Files:** `tests/a.test.mjs`, `src/a.mjs`.\n- **Changes:** x\n';
|
|
assert.equal(stepTestsOnly(body).testsOnly, false, 'a step that changes src/a.mjs is not tests-only');
|
|
const { plannedPaths } = await import('../../lib/verification/plan-scope-sweep.mjs');
|
|
const plan = '## Implementation Plan\n\n### Step 1: x\n\n- **Files:** `src/cli.mjs`: add the flag; `src/b.mjs`.\n- **Changes:** x\n';
|
|
assert.deepEqual(plannedPaths(plan), ['src/b.mjs', 'src/cli.mjs'], 'K6 plans what v6.2.0 planned');
|
|
});
|
|
|
|
test('review 630 r4: a quoted frontmatter value may carry a YAML comment', async () => {
|
|
const { readFrontmatterFields } = await import('../../lib/skills/skill-frontmatter.mjs');
|
|
const r = readFrontmatterFields('---\nname: "debugging" # pinned\ndescription: \'Use it.\' # note\n---\nbody\n');
|
|
assert.equal(r.fields.name?.value, 'debugging', JSON.stringify(r));
|
|
assert.equal(r.fields.description?.value, 'Use it.', JSON.stringify(r));
|
|
const bad = readFrontmatterFields('---\nname: "debugging" trailing\n---\nbody\n');
|
|
assert.ok(JSON.stringify(bad).includes('unclosed double quote') || bad.fields.name?.value !== 'debugging',
|
|
'text after the closing quote that is not a comment is not accepted silently');
|
|
});
|
|
|
|
test('review 630 r4: no tilde word, and nothing after `make test`', async () => {
|
|
const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
|
for (const cmd of ['node --test ~/x.test.mjs', 'node --test --test-name-pattern=~x', 'git log ~/x', 'git diff ~root/x',
|
|
'make test install', 'make test deploy']) {
|
|
assert.equal(allowedCommand(cmd, { cwd: ROOT }).allowed, false, `${cmd} must not reach the shell`);
|
|
}
|
|
for (const cmd of ['git diff --stat HEAD~1', 'git show HEAD~2:package.json', 'make test']) {
|
|
assert.equal(allowedCommand(cmd, { cwd: ROOT }).allowed, true, `the control: ${cmd}`);
|
|
}
|
|
});
|
|
|
|
// ==== round 5 of the release review (2026-10-11), each red on d37e538 ===========
|
|
// Operator 2026-10-11: no more patching of the same two places. A check runs
|
|
// with no shell (its words reach the runner as written), and the K6 sweep has
|
|
// its own v6.2.0 reader again; the shared reader stays for the tests-only check.
|
|
|
|
const PASS_SCREENS = { allow: () => ({ allowed: true, reason: '' }), screen: () => ({ allowed: true, rule: '' }) };
|
|
|
|
test('review 630 r5 / da7584e7: a check runs without a shell - its words reach the runner as written', async () => {
|
|
const { runCriteria } = await import('../../lib/verification/criteria-runner.mjs');
|
|
const dir = mkdtempSync(join(tmpdir(), 'r630-noshell-'));
|
|
try {
|
|
mkdirSync(join(dir, 'tests'));
|
|
writeFileSync(join(dir, 'tests', 'args.sh'), 'printf "%s\\n" "$@"\n');
|
|
const [r] = runCriteria([{ label: 'X1', text: 'x', command: 'bash tests/args.sh ~/x *.sh $HOME' }], { cwd: dir, ...PASS_SCREENS });
|
|
assert.equal(r.status, 'passed', r.output);
|
|
assert.equal(r.output, '~/x\n*.sh\n$HOME\n', 'no shell expanded a tilde, a glob or a variable');
|
|
const [missing] = runCriteria([{ label: 'X2', text: 'x', command: 'no-such-runner-r630 --x' }], { cwd: dir, ...PASS_SCREENS });
|
|
// Round 6: a runner that could not start is NOT RUN (no measurement), never a pass and never a red.
|
|
assert.equal(missing.status, 'unrunnable', 'a runner that is not installed is not run, never a pass');
|
|
assert.match(missing.output, /could not start.*ENOENT/i, missing.output);
|
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('review 630 r5 / c25f8704: a glob cannot climb out of tests/ or the repo', async () => {
|
|
const { runCriteria, allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
|
const base = mkdtempSync(join(tmpdir(), 'r630-escape-'));
|
|
const repo = join(base, 'repo');
|
|
try {
|
|
mkdirSync(join(repo, 'tests'), { recursive: true });
|
|
mkdirSync(join(repo, 'outside'));
|
|
writeFileSync(join(repo, 'outside', 'evil.sh'), 'echo ESCAPED > escaped-bash.txt\n');
|
|
writeFileSync(join(repo, 'tests', 'a.test.mjs'), "import { test } from 'node:test';\ntest('in', () => {});\n");
|
|
writeFileSync(join(base, 'outside.test.mjs'),
|
|
`import { test } from 'node:test';\nimport { writeFileSync } from 'node:fs';\ntest('out', () => { writeFileSync(${JSON.stringify(join(base, 'escaped-node.txt'))}, 'x'); });\n`);
|
|
const results = runCriteria([
|
|
{ label: 'E1', text: 'x', command: 'bash tests/.?/outside/evil.sh' },
|
|
{ label: 'E2', text: 'x', command: 'node --test .*' },
|
|
], { cwd: repo });
|
|
assert.equal(existsSync(join(repo, 'escaped-bash.txt')), false, `bash ran a script outside tests/: ${JSON.stringify(results[0])}`);
|
|
assert.equal(existsSync(join(base, 'escaped-node.txt')), false, `node --test ran a test file outside the repo: ${JSON.stringify(results[1])}`);
|
|
assert.notEqual(results[0].status, 'passed');
|
|
for (const cmd of ['node --test ../outside.test.mjs', 'node --test tests/../../outside.test.mjs', 'node --test /tmp/x.test.mjs',
|
|
'pytest ../tests', 'go test ../...', 'npm test -- ../x.test.mjs', 'jest /abs/x.test.js', 'vitest run ../x', 'bash tests/*.sh',
|
|
'pytest tests/test_*.py', 'jest src/*.test.js', 'npm test -- tests/*.mjs']) {
|
|
assert.equal(allowedCommand(cmd, { cwd: ROOT }).allowed, false, `${cmd} must not run`);
|
|
}
|
|
for (const cmd of ['node --test tests/lib/*.test.mjs', 'node --test tests/**/*.test.mjs', 'go test ./...', '/bin/bash tests/fixtures/criteria-exit-0.sh',
|
|
'node --test tests/', 'pytest tests/', 'git diff HEAD~1..HEAD --stat']) {
|
|
const r = allowedCommand(cmd, { cwd: ROOT });
|
|
assert.equal(r.allowed, true, `the control: ${cmd} (${r.reason})`);
|
|
}
|
|
} finally { rmSync(base, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('review 630 r5 / 8f3cf84d: a word starting with @ is refused (pytest reads @file as more arguments)', async () => {
|
|
const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
|
for (const cmd of ['pytest @tests/args.txt', 'python -m pytest @tests/args.txt', 'python3 -m pytest -q @a', 'uv run pytest @a', 'node --test @a', 'jest @a']) {
|
|
const r = allowedCommand(cmd, { cwd: ROOT });
|
|
assert.equal(r.allowed, false, `${cmd} must not run`);
|
|
assert.match(r.reason, /@/);
|
|
}
|
|
});
|
|
|
|
test('review 630 r5 / da7584e7: every place that runs a check runs it with no shell', () => {
|
|
for (const rel of ['lib/verification/criteria-runner.mjs', 'lib/verification/regression-check.mjs', 'lib/verification/red-proof.mjs',
|
|
'lib/verification/done-line.mjs', 'lib/util/test-lock.mjs']) {
|
|
const src = read(rel);
|
|
assert.doesNotMatch(src, /spawnSync\(\s*['"](?:\/bin\/)?(?:ba)?sh['"]/, `${rel} starts a shell`);
|
|
assert.doesNotMatch(src, /shell:\s*true/, `${rel} asks spawn for a shell`);
|
|
if (rel !== 'lib/verification/criteria-runner.mjs') assert.match(src, /\brunCommand\b/, `${rel} runs through criteria-runner's runCommand`);
|
|
}
|
|
assert.doesNotMatch(read('lib/verification/regression-check.mjs'), /function exec\(/, 'regression-check has no exec of its own (MINOR 0e98d5c7)');
|
|
});
|
|
|
|
test('review 630 r5 / da7584e7: the done line runs its command with no shell', async () => {
|
|
// sh reads `**` as `*` (one level); node --test expands its own pattern at
|
|
// every depth. Only with no shell does the top-level test run.
|
|
const { writeDoneLine } = await import('../../lib/verification/done-line.mjs');
|
|
const repo = mkdtempSync(join(tmpdir(), 'r630-done-'));
|
|
try {
|
|
mkdirSync(join(repo, 'tests', 'sub'), { recursive: true });
|
|
writeFileSync(join(repo, 'tests', 'sub', 'deep.test.mjs'), "import { test } from 'node:test';\ntest('deep', () => {});\n");
|
|
writeFileSync(join(repo, 'tests', 'top.test.mjs'),
|
|
`import { test } from 'node:test';\nimport { writeFileSync } from 'node:fs';\ntest('top', () => { writeFileSync(${JSON.stringify(join(repo, 'top-ran.txt'))}, 'x'); });\n`);
|
|
for (const args of [['init', '-q'], ['-c', 'user.email=t@example.invalid', '-c', 'user.name=t', 'commit', '-q', '--allow-empty', '-m', 'x']]) {
|
|
assert.equal(spawnSync('git', args, { cwd: repo }).status, 0);
|
|
}
|
|
const r = writeDoneLine({ cwd: repo, step: 1, cmd: 'node --test tests/**/*.test.mjs', report: join(repo, 'report.md') });
|
|
assert.equal(r.status, 'written', JSON.stringify(r));
|
|
assert.equal(existsSync(join(repo, 'top-ran.txt')), true, 'the pattern reached node as written');
|
|
} finally { rmSync(repo, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('review 630 r5 / ff80c7ef 5dc53c7e 02cea4aa: the K6 sweep plans what v6.2.0 planned, with its own reader', async () => {
|
|
const { plannedPaths } = await import('../../lib/verification/plan-scope-sweep.mjs');
|
|
const plan = (files) => `## Implementation Plan\n\n### Step 1: x\n\n- **Files:** ${files}\n- **Changes:** x\n`;
|
|
const cases = [
|
|
['`src/a.mjs` — replaces the helper in lib/util.mjs and touches config.json; see Node.js docs', ['src/a.mjs']],
|
|
['`src/cli.mjs` — follows the pattern in lib/', ['src/cli.mjs']],
|
|
['.gitignore, src/a.mjs', ['.gitignore', 'src/a.mjs']],
|
|
['src\\cli.mjs', ['src/cli.mjs']],
|
|
['`src/a.mjs`/`src/b.mjs`', ['src/a.mjs', 'src/b.mjs']],
|
|
['`src/a.mjs`—adds', ['src/a.mjs']],
|
|
['`src/cli.mjs`: add the flag; `src/b.mjs`.', ['src/b.mjs', 'src/cli.mjs']],
|
|
];
|
|
for (const [files, want] of cases) assert.deepEqual(plannedPaths(plan(files)), want, files);
|
|
assert.doesNotMatch(read('lib/verification/plan-scope-sweep.mjs'), /import \{[^}]*\bfieldPaths\b/, 'K6 does not read Files: through the tests-only reader');
|
|
});
|
|
|
|
test('review 630 r5 / a4c8005d 02cea4aa: the tests-only check takes only what passes a path test', () => {
|
|
const step = (files) => `- **Files:** ${files}\n- **Changes:** x\n`;
|
|
const cases = [
|
|
['`tests/parse.test.mjs` (asserts `parse()` throws)', true, ['tests/parse.test.mjs']],
|
|
['`tests/a.test.mjs` — covers pass/fail', true, ['tests/a.test.mjs']],
|
|
['`tests/a.test.mjs` — covers JSON, YAML', true, ['tests/a.test.mjs']],
|
|
['`tests/a.test.mjs` — calls `parser.parse()` twice', true, ['tests/a.test.mjs']],
|
|
['`tests/a.test.mjs`, .gitignore', false, ['tests/a.test.mjs', '.gitignore']],
|
|
['tests/a.test.mjs, .gitignore', false, ['tests/a.test.mjs', '.gitignore']],
|
|
['`tests/a.test.mjs`/`src/b.mjs`', false, ['tests/a.test.mjs', 'src/b.mjs']],
|
|
['`tests\\a.test.mjs`', true, ['tests/a.test.mjs']],
|
|
['tests\\a.test.mjs, src\\cli.mjs', false, ['tests/a.test.mjs', 'src/cli.mjs']],
|
|
['Makefile.', false, ['Makefile']],
|
|
];
|
|
for (const [files, testsOnly, want] of cases) assert.deepEqual(stepTestsOnly(step(files)), { testsOnly, files: want }, files);
|
|
});
|
|
|
|
test('review 630 r5 / e1ff583a f582b9c2: frontmatter reads trailing space and a comment after a plain value as YAML does', async () => {
|
|
const { readFrontmatterFields } = await import('../../lib/skills/skill-frontmatter.mjs');
|
|
const r = readFrontmatterFields('---\nname: "debugging" \nmodel: opus # operator pin\neffort: high # pinned\ntags: [a] \n' +
|
|
'allowed-tools: [Read, Grep] # read only\ndescription: | # folded below\n Use when x.\n---\nbody\n');
|
|
assert.deepEqual(r.hazards, [], JSON.stringify(r.hazards));
|
|
assert.equal(r.fields.name.value, 'debugging');
|
|
assert.equal(r.fields.model.value, 'opus');
|
|
assert.equal(r.fields.effort.value, 'high');
|
|
assert.deepEqual(r.fields.tags.value, ['a']);
|
|
assert.deepEqual(r.fields['allowed-tools'].value, ['Read', 'Grep']);
|
|
assert.equal(r.fields.description.value, 'Use when x.\n');
|
|
// The control: a description is the text Claude Code routes on, so a comment
|
|
// that cuts it is still a hazard (quote it), and so is a value that is only a comment.
|
|
for (const fm of ['description: Use when x happens #1.', 'description: Use when x # note', 'name: # x']) {
|
|
assert.ok(readFrontmatterFields(`---\n${fm}\n---\nbody\n`).hazards.length > 0, fm);
|
|
}
|
|
});
|
|
|
|
test('review 630 r5 / 41c43285 (operator 2026-10-09): the example plan keeps steps 1-2, and step 2 no longer says it must fail', () => {
|
|
const plan = read('examples/01-add-verbose-flag/plan.md');
|
|
const heading = plan.match(/^### Step 2: .*$/m)[0];
|
|
assert.doesNotMatch(heading, /must FAIL/i, heading);
|
|
assert.match(plan.slice(plan.indexOf(heading)), /should PASS today/, 'the step body still says it passes today');
|
|
const progress = JSON.parse(read('examples/01-add-verbose-flag/progress.json'));
|
|
const title = JSON.stringify(progress).match(/"title":"(Add stability test[^"]*)"/)?.[1];
|
|
assert.equal(`### Step 2: ${title}`, heading, 'progress.json carries the same step title');
|
|
const r = validatePlanContent(plan, { strict: false });
|
|
const warned = r.warnings.filter((w) => w.code === 'PLAN_STEP_TESTS_ONLY').map((w) => w.message);
|
|
assert.equal(warned.length, 2, `the operator kept the warning on steps 1-2: ${JSON.stringify(warned)}`);
|
|
});
|
|
|
|
// ==== round 6 of the release review (2026-10-11), each red on c3641ff ===========
|
|
|
|
function gitRepo(prefix) {
|
|
const dir = mkdtempSync(join(tmpdir(), prefix));
|
|
for (const args of [['init', '-q'], ['config', 'user.email', 't@example.invalid'], ['config', 'user.name', 't'], ['config', 'commit.gpgsign', 'false']]) {
|
|
assert.equal(spawnSync('git', args, { cwd: dir }).status, 0);
|
|
}
|
|
return dir;
|
|
}
|
|
const commitAll = (dir, msg) => {
|
|
assert.equal(spawnSync('git', ['add', '-A'], { cwd: dir }).status, 0);
|
|
assert.equal(spawnSync('git', ['commit', '-q', '-m', msg], { cwd: dir }).status, 0);
|
|
return spawnSync('git', ['rev-parse', 'HEAD'], { cwd: dir, encoding: 'utf8' }).stdout.trim();
|
|
};
|
|
|
|
test('review 630 r6 / criteria-runner.mjs:464: a git check stays in the repo too', async () => {
|
|
const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
|
for (const cmd of ['git diff /etc/hosts /dev/null', 'git diff ../x tests/a.test.mjs', 'git show HEAD:../x', 'git log -- /etc',
|
|
'git diff --no-index tests/a tests/b']) {
|
|
assert.equal(allowedCommand(cmd, { cwd: ROOT }).allowed, false, `${cmd} must not run`);
|
|
}
|
|
for (const cmd of ['git diff HEAD~1..HEAD --stat', 'git show HEAD:package.json', 'git log -- lib/', 'git show @~1 --stat']) {
|
|
const r = allowedCommand(cmd, { cwd: ROOT });
|
|
assert.equal(r.allowed, true, `the control: ${cmd} (${r.reason})`);
|
|
}
|
|
});
|
|
|
|
test('review 630 r6 / criteria-runner.mjs:168: a trailing # comment in a shell fence is not part of the command', async () => {
|
|
const { parsePlanVerification, allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
|
const plan = '## Verification\n\n```bash\nnpm test # all green\nnode --test tests/a.test.mjs #1\n```\n';
|
|
assert.deepEqual(parsePlanVerification(plan).map((c) => c.command), ['npm test', 'node --test tests/a.test.mjs']);
|
|
const example = parsePlanVerification(read('examples/01-add-verbose-flag/plan.md'))[0];
|
|
assert.equal(example.command, 'node --test tests/');
|
|
assert.equal(allowedCommand(example.command, { cwd: ROOT }).allowed, true, "the shipped example's first check runs again");
|
|
});
|
|
|
|
test('review 630 r6 / criteria-runner.mjs:466: a glob reaches node --test only where node expands it (Node 21+)', async () => {
|
|
const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
|
const old = allowedCommand('node --test tests/lib/*.test.mjs', { cwd: ROOT, nodeMajor: 20 });
|
|
assert.equal(old.allowed, false);
|
|
assert.match(old.reason, /21/);
|
|
assert.equal(allowedCommand('node --test tests/lib/*.test.mjs', { cwd: ROOT, nodeMajor: 22 }).allowed, true);
|
|
assert.equal(allowedCommand('node --test tests/lib/review-630.test.mjs', { cwd: ROOT, nodeMajor: 20 }).allowed, true, 'the control: no glob, any Node');
|
|
});
|
|
|
|
test('review 630 r6 / plan-schema.mjs:318: a backticked file name with no extension is a path', () => {
|
|
const step = (files) => `- **Files:** ${files}\n- **Changes:** x\n`;
|
|
assert.deepEqual(stepTestsOnly(step('`tests/build.test.mjs`, `Makefile`')), { testsOnly: false, files: ['tests/build.test.mjs', 'Makefile'] });
|
|
assert.deepEqual(stepTestsOnly(step('`spec/a_spec.rb`, `Gemfile`')).testsOnly, false);
|
|
assert.deepEqual(stepTestsOnly(step('`tests/a.test.mjs` — covers `parse` and `JSON`')).files, ['tests/a.test.mjs'], 'a span inside a description is no path');
|
|
});
|
|
|
|
// Round 6 had red proof overrule the plan with the step's diff; the operator
|
|
// reversed that on 2026-10-11 (round 7): the plan's Files: alone decides, and a
|
|
// plain Makefile beside a backticked test is now read from Files: itself.
|
|
test('review 630 r6 / plan-schema.mjs:329 (reversed in r7): a Makefile beside a backticked test is read from Files:, not from the diff', async () => {
|
|
const { proveRed } = await import('../../lib/verification/red-proof.mjs');
|
|
const work = mkdtempSync(join(tmpdir(), 'r630-diff-'));
|
|
const repo = gitRepo('r630-diff-repo-');
|
|
try {
|
|
writeFileSync(join(repo, 'README.md'), 'x\n');
|
|
const base = commitAll(repo, 'base');
|
|
mkdirSync(join(repo, 'tests'));
|
|
writeFileSync(join(repo, 'tests', 'a.test.mjs'), "import { test } from 'node:test';\ntest('a', () => {});\n");
|
|
writeFileSync(join(repo, 'Makefile'), 'test:\n\tnode --test tests/\n');
|
|
const plan = join(work, 'plan.md');
|
|
writeFileSync(plan, '## Implementation Plan\n\n### Step 1: x\n\n- **Files:** `tests/a.test.mjs`, Makefile\n- **Changes:** x\n');
|
|
const o = { cwd: repo, base, cmd: 'node --test tests/a.test.mjs', tests: ['tests/a.test.mjs'], plan, step: 1 };
|
|
const r = proveRed(o);
|
|
assert.equal(r.reason, 'not-red-at-base', JSON.stringify(r));
|
|
assert.equal(r.tests_only, false, 'Files: names Makefile');
|
|
rmSync(join(repo, 'Makefile'));
|
|
assert.equal(proveRed(o).tests_only, false, 'the diff is not read: without Makefile in the tree the plan still names it');
|
|
} finally { rmSync(work, { recursive: true, force: true }); rmSync(repo, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('review 630 r6 / red-proof.mjs:129: exit 127 from the test program itself is a red, not "could not run"', async () => {
|
|
const { proveRed } = await import('../../lib/verification/red-proof.mjs');
|
|
const { writeDoneLine } = await import('../../lib/verification/done-line.mjs');
|
|
const repo = gitRepo('r630-127-');
|
|
try {
|
|
writeFileSync(join(repo, 'README.md'), 'x\n');
|
|
const base = commitAll(repo, 'base');
|
|
mkdirSync(join(repo, 'tests'));
|
|
mkdirSync(join(repo, 'bin'));
|
|
writeFileSync(join(repo, 'tests', 'e2e.sh'), './bin/tool\n');
|
|
writeFileSync(join(repo, 'bin', 'tool'), '#!/bin/sh\nexit 0\n', { mode: 0o755 });
|
|
const r = proveRed({ cwd: repo, base, cmd: 'bash tests/e2e.sh', tests: ['tests/e2e.sh'], redOnly: true });
|
|
assert.equal(r.status, 'proven', JSON.stringify(r));
|
|
assert.equal(r.base.exit, 127);
|
|
rmSync(join(repo, 'bin'), { recursive: true });
|
|
const d = writeDoneLine({ cwd: repo, step: 1, cmd: 'bash tests/e2e.sh', report: join(repo, 'report.md') });
|
|
assert.equal(d.status, 'failed', `a test that exits 127 failed: ${JSON.stringify(d)}`);
|
|
} finally { rmSync(repo, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('review 630 r6 / regression-check.mjs:64: a Regression: whose runner cannot start is not-run, not a red suite', async () => {
|
|
const { checkRegression } = await import('../../lib/verification/regression-check.mjs');
|
|
const empty = mkdtempSync(join(tmpdir(), 'r630-path-'));
|
|
const saved = process.env.PATH;
|
|
try {
|
|
process.env.PATH = empty;
|
|
const planText = '## Implementation Plan\n\n### Step 1: x\n\n- **Files:** `src/a.mjs`\n- **Regression:** `vitest run`\n';
|
|
const r = checkRegression({ planText, step: 1, cwd: ROOT });
|
|
assert.equal(r.status, 'error', JSON.stringify(r));
|
|
assert.equal(r.progress_status, 'not-run');
|
|
assert.equal(r.exit, 2);
|
|
assert.match(r.output, /could not start/);
|
|
} finally { process.env.PATH = saved; rmSync(empty, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('review 630 r6 / hooks are ASCII: the lines this release added to pre-bash-executor.mjs', () => {
|
|
const src = read('hooks/scripts/pre-bash-executor.mjs');
|
|
const block = src.slice(src.indexOf('Everything this guard would ask about') - 200, src.indexOf('Everything this guard would ask about') + 300);
|
|
assert.ok(block.includes('Everything this guard would ask about'));
|
|
assert.doesNotMatch(block, /[^\x00-\x7f]/, 'a hook script is ASCII (~/.claude/CLAUDE.md, Miljø)');
|
|
});
|
|
|
|
// ==== round 7 of the release review (2026-10-11), fixed in 6.3.1; each red on 0f400d3 ====
|
|
|
|
test('review 630 r7 / 48939bcb d20c5619 (operator 2026-10-11): the plan\'s Files: alone decides tests-only, whatever else is in the tree', async () => {
|
|
const { proveRed } = await import('../../lib/verification/red-proof.mjs');
|
|
const repo = gitRepo('r630-r7-files-');
|
|
try {
|
|
writeFileSync(join(repo, 'README.md'), 'x\n');
|
|
const base = commitAll(repo, 'base');
|
|
// Work the plan does not know about, there before the step began (trekexecute
|
|
// Sub-step 0 lets a step start on a dirty tree and logs it as a decision).
|
|
writeFileSync(join(repo, 'README.md'), 'edited before the step\n');
|
|
writeFileSync(join(repo, 'notes.txt'), 'untracked, not ignored\n');
|
|
// A plan placed outside .claude/, with the executor's files beside it (brief note 2).
|
|
mkdirSync(join(repo, 'plans', 'steps'), { recursive: true });
|
|
const plan = join(repo, 'plans', 'plan.md');
|
|
writeFileSync(plan, '## Implementation Plan\n\n### Step 1: x\n\n- **Files:** `tests/a.test.mjs`\n- **Changes:** x\n\n' +
|
|
'### Step 2: y\n\n- **Files:** `tests/a.test.mjs`, `src/a.mjs`\n- **Changes:** y\n');
|
|
writeFileSync(join(repo, 'plans', 'progress.json'), '{}\n');
|
|
writeFileSync(join(repo, 'plans', 'steps', 'step-1.report.md'), 'report\n');
|
|
// The helper wrote the step's test alone (step-implementer, mode build, step 1); it passes at the start point.
|
|
mkdirSync(join(repo, 'tests'));
|
|
writeFileSync(join(repo, 'tests', 'a.test.mjs'), "import { test } from 'node:test';\ntest('a', () => {});\n");
|
|
const o = { cwd: repo, base, cmd: 'node --test tests/a.test.mjs', tests: ['tests/a.test.mjs'], plan };
|
|
for (const redOnly of [true, false]) {
|
|
const one = proveRed({ ...o, step: 1, redOnly });
|
|
assert.equal(one.reason, 'not-red-at-base', JSON.stringify(one));
|
|
assert.equal(one.tests_only, true, `step 1's Files: names only its test (redOnly ${redOnly}): ${JSON.stringify(one.tests_only_note ?? null)}`);
|
|
const two = proveRed({ ...o, step: 2, redOnly });
|
|
assert.equal(two.reason, 'not-red-at-base', JSON.stringify(two));
|
|
assert.equal(two.tests_only, false, `step 2's Files: names src/a.mjs, though the tree holds only its test (redOnly ${redOnly})`);
|
|
}
|
|
} finally { rmSync(repo, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('review 630 r7 / 48939bcb (operator 2026-10-11): the text says plainly that the plan\'s Files: alone decides', () => {
|
|
for (const rel of ['commands/trekexecute.md', 'agents/step-implementer.md', 'lib/verification/red-proof.mjs', 'docs/HANDOVER-CONTRACTS.md']) {
|
|
const text = read(rel);
|
|
assert.match(text, /`Files:` alone decides/, `${rel} says that the plan's Files: alone decides`);
|
|
assert.doesNotMatch(text, /every file (?:the step|it) changed since (?:its|the) start point/, `${rel} no longer reads the step's diff for tests-only`);
|
|
}
|
|
const row = read('commands/trekexecute.md').split('\n').find((l) => l.startsWith('| 1, `not-red-at-base`, `tests_only: true`'));
|
|
assert.match(row, /`Files:` alone decides/, 'the routing row itself says it');
|
|
});
|
|
|
|
test('review 630 r7 / 0180a979: a file named without backticks beside a backticked path is read', () => {
|
|
const step = (files) => `- **Files:** ${files}\n- **Changes:** x\n`;
|
|
for (const [files, paths] of [
|
|
['`tests/x.test.mjs`, Dockerfile', ['tests/x.test.mjs', 'Dockerfile']],
|
|
['`tests/x.test.mjs`, the CLI entry src/cli.mjs', ['tests/x.test.mjs', 'src/cli.mjs']],
|
|
['`tests/x.test.mjs` and Makefile', ['tests/x.test.mjs', 'Makefile']],
|
|
['`tests/x.test.mjs` and `Makefile`', ['tests/x.test.mjs', 'Makefile']],
|
|
]) {
|
|
assert.deepEqual(stepTestsOnly(step(files)), { testsOnly: false, files: paths }, files);
|
|
}
|
|
// The control: a description beside a backticked path is still no path.
|
|
for (const files of ['`tests/a.test.mjs` — covers `parse` and `JSON`', '`tests/a.test.mjs` — covers pass/fail, JSON, YAML',
|
|
'`tests/a.test.mjs` (v6.3.0 format)', '`tests/a.test.mjs`: Covers the Parser, e.g. its Profile']) {
|
|
assert.deepEqual(stepTestsOnly(step(files)), { testsOnly: true, files: ['tests/a.test.mjs'] }, files);
|
|
}
|
|
});
|
|
|
|
test('review 630 r7 / 633546914: a command its timeout killed is reported as killed by the timeout, not as "could not start"', async () => {
|
|
const { proveRed } = await import('../../lib/verification/red-proof.mjs');
|
|
const repo = gitRepo('r630-r7-hang-');
|
|
try {
|
|
writeFileSync(join(repo, 'README.md'), 'x\n');
|
|
const base = commitAll(repo, 'base');
|
|
mkdirSync(join(repo, 'tests'));
|
|
writeFileSync(join(repo, 'tests', 'hang.test.mjs'), "import { test } from 'node:test';\ntest('hangs', () => new Promise((r) => setTimeout(r, 6000)));\n");
|
|
const r = proveRed({ cwd: repo, base, cmd: 'node --test tests/hang.test.mjs', tests: ['tests/hang.test.mjs'], redOnly: true, timeoutMs: 2000 });
|
|
assert.equal(r.status, 'not-run', JSON.stringify(r));
|
|
assert.doesNotMatch(r.reason, /could not start/, r.reason);
|
|
assert.match(r.reason, /killed .*after the 2 s timeout/, r.reason);
|
|
} finally { rmSync(repo, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('review 630 r7 / 5b2063e2: a trailing # comment is no part of a backticked check either, and never reaches the runner', async () => {
|
|
const { firstCommand, allowedCommand, runCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
|
assert.equal(firstCommand('`npm test # full suite`').command, 'npm test');
|
|
assert.equal(firstCommand('`node --test tests/a.test.mjs #1` — expected: pass').command, 'node --test tests/a.test.mjs');
|
|
const plan = '## Implementation Plan\n\n### Step 1: x\n\n- **Files:** `src/a.mjs`\n- **Changes:** x\n' +
|
|
'- **Regression:** `npm test # full suite`\n';
|
|
const r = validatePlanContent(plan, { strict: false });
|
|
const codes = [...r.errors, ...r.warnings].map((i) => i.code);
|
|
assert.ok(!codes.includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'), JSON.stringify(codes));
|
|
// The executor passes the Verify span to red proof, the done line and the test lock as written.
|
|
assert.equal(allowedCommand('npm test # full suite', { cwd: ROOT }).allowed, true);
|
|
assert.equal(runCommand('node -e process.exit(process.argv.length-1) # two words', { cwd: ROOT }).status, 0, 'the comment\'s words reach no program');
|
|
assert.equal(firstCommand('`node --test tests/a#b.test.mjs`').command, 'node --test tests/a#b.test.mjs', 'the control: a # inside a word is no comment');
|
|
});
|
|
|
|
test('review 630 r7 / aa08f47f: the 15-minute timeout is defined once', async () => {
|
|
const files = spawnSync('git', ['ls-files', 'lib', 'scripts', 'hooks'], { cwd: ROOT, encoding: 'utf8' }).stdout.split('\n').filter((p) => p.endsWith('.mjs'));
|
|
assert.ok(files.length > 50, `listed ${files.length} files`);
|
|
assert.deepEqual(files.filter((p) => /\b900_?000\b/.test(read(p))), ['lib/verification/criteria-runner.mjs']);
|
|
const rp = await import('../../lib/verification/red-proof.mjs');
|
|
const cr = await import('../../lib/verification/criteria-runner.mjs');
|
|
assert.equal(rp.DEFAULT_TIMEOUT_MS, cr.DEFAULT_TIMEOUT_MS);
|
|
});
|
|
|
|
test('review 630 r7 / efa7de84: a file cited many times is read once per validation', () => {
|
|
const cite = 'commands/trekexecute.md:1';
|
|
const findings = Array.from({ length: 40 }, (_, i) => ({
|
|
severity: 'MINOR', rule_key: 'MISSING_TEST', file: 'commands/trekexecute.md', line: 1 + i,
|
|
title: `see ${cite} and ${cite}0`, detail: `${cite}2`, recommended_action: 'Add a test.',
|
|
}));
|
|
let reads = 0;
|
|
const real = fs.readFileSync;
|
|
mock.method(fs, 'readFileSync', function (p, ...rest) {
|
|
if (String(p).endsWith(`${'/'}commands/trekexecute.md`)) reads++;
|
|
return real.call(this, p, ...rest);
|
|
});
|
|
syncBuiltinESMExports();
|
|
let r;
|
|
try { r = validateFindings({ reviewer: 'r', findings }, { repoRoot: ROOT }); }
|
|
finally { mock.restoreAll(); syncBuiltinESMExports(); }
|
|
assert.equal(r.valid, true, JSON.stringify(r.errors));
|
|
assert.equal(reads, 1, `commands/trekexecute.md was read ${reads} times for one validation`);
|
|
});
|
|
|
|
test('review 630 r7 / d5ab7698: one isMain helper, and a CLI started through a symlinked path with a space does its work', () => {
|
|
const files = spawnSync('git', ['ls-files', 'lib', 'scripts', 'hooks'], { cwd: ROOT, encoding: 'utf8' }).stdout.split('\n').filter((p) => p.endsWith('.mjs'));
|
|
const local = files.filter((p) => p !== 'lib/util/is-main.mjs' &&
|
|
/function isMain\b|import\.meta\.url === `file:\/\/\$\{process\.argv\[1\]\}`/.test(read(p)));
|
|
assert.deepEqual(local, [], 'every CLI uses lib/util/is-main.mjs');
|
|
const base = mkdtempSync(join(tmpdir(), 'r630 r7 main '));
|
|
const link = join(base, 'voyage link');
|
|
try {
|
|
symlinkSync(ROOT, link);
|
|
// The first had the file:// guard (a silent exit 0 here), the second the realpath copy.
|
|
for (const cli of ['lib/validators/session-state-validator.mjs', 'lib/validators/plan-validator.mjs']) {
|
|
const r = spawnSync(process.execPath, [join(link, cli)], { encoding: 'utf8', input: '', timeout: 30_000 });
|
|
assert.equal(r.status, 2, `${cli} through "${link}": ${r.stdout}${r.stderr}`);
|
|
assert.match(r.stderr, /usage/i);
|
|
}
|
|
} finally { fs.rmSync(link, { force: true }); rmSync(base, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('review 630 r7 / 4f8a69e7: the guard writes its no-one-to-ask refusal once, on a direct condition', () => {
|
|
const src = read('hooks/scripts/pre-bash-executor.mjs');
|
|
assert.doesNotMatch(src, /const unattended = /);
|
|
assert.equal(src.split('permission mode where Claude Code asks nothing').length - 1, 1, 'one refusal text for both cases');
|
|
});
|
|
|
|
test('review 630 r7 / findings-schema.mjs:41: a triple backtick inside a JSON string does not end the block', async () => {
|
|
const { extractFindingsBlock } = await import('../../lib/review/findings-schema.mjs');
|
|
const { fillFindingIds } = await import('../../lib/review/fill-finding-ids.mjs');
|
|
const payload = { verdict: 'WARN', findings: [{ id: '@@ID_1@@', severity: 'MAJOR', rule_key: 'MISSING_TEST', file: 'a.mjs', line: 1,
|
|
title: 'x', detail: 'write the check as ```bash\nnpm test\n``` and run it', recommended_action: 'Add a test.' }] };
|
|
const text = `# Review\n\nProse with a \`\`\`json fence quoted in it.\n\n\`\`\`json\n${JSON.stringify(payload, null, 2)}\n\`\`\`\n`;
|
|
const block = extractFindingsBlock(text);
|
|
assert.match(block, /"recommended_action": "Add a test\."\s*\}\s*\]\s*\}\s*$/, `the block runs to its own closing fence, not to the backticks in a string: …${block.slice(-60)}`);
|
|
assert.deepEqual(JSON.parse(block), payload);
|
|
const r = fillFindingIds(text);
|
|
assert.equal(r.ok, true, JSON.stringify(r.errors));
|
|
assert.doesNotMatch(r.text, /@@ID_1@@/);
|
|
});
|
|
|
|
test('review 630 r7 / brief note 3: the prose says a git revision may start with @, as the screen allows', async () => {
|
|
const { allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
|
assert.equal(allowedCommand('git show @~1 --stat', { cwd: ROOT }).allowed, true);
|
|
assert.equal(allowedCommand('pytest @args.txt', { cwd: ROOT }).allowed, false, 'the control: a runner word with a leading @');
|
|
for (const rel of ['agents/brief-reviewer.md', 'commands/trekplan.md']) {
|
|
assert.match(read(rel), /`@~1`/, `${rel} names the git revision as the exception`);
|
|
}
|
|
});
|
|
|
|
// ==== round 8 of the release review of 6.3.0 (0f400d3..29b4ac9, 2026-10-11), fixed in 6.3.1 ====
|
|
|
|
test('review 630 r8 / BLOCKER criteria-runner.mjs:158: a Verify comment that a shell would read inside double quotes is refused', () => {
|
|
// The executor and the helper hand the Verify span to red proof, the test lock
|
|
// and the done line as `--cmd "{Verify command}"`: inside double quotes a `$`,
|
|
// a backtick or a backslash still means something, and a `"` ends the argument.
|
|
const verify = (span) => buildPlan().replace('`node --test tests/mod1.test.mjs` → expected: pass', () => `\`${span}\` → expected: pass`);
|
|
const codes = (span) => {
|
|
const r = validatePlanContent(verify(span), { strict: false });
|
|
return [...r.errors, ...r.warnings].filter((i) => i.code === 'PLAN_STEP_VERIFY_NOT_RUNNABLE').map((i) => i.message);
|
|
};
|
|
for (const span of ['node --test tests/mod1.test.mjs # $(touch x)', 'node --test tests/mod1.test.mjs # expect "fail 1"',
|
|
'node --test tests/mod1.test.mjs # a back\\slash', 'node --test tests/mod1.test.mjs # ${HOME}']) {
|
|
assert.equal(codes(span).length, 1, `${span} is refused: ${JSON.stringify(codes(span))}`);
|
|
}
|
|
// The control: a plain comment is still dropped, and the check stays runnable.
|
|
assert.deepEqual(codes('node --test tests/mod1.test.mjs # all green'), []);
|
|
assert.deepEqual(codes("node --test tests/mod1.test.mjs # it's green"), [], 'a single quote means nothing inside double quotes');
|
|
});
|
|
|
|
test('review 630 r8 / criteria-runner.mjs:158: a span that is only a comment is not-a-command, not a placeholder', async () => {
|
|
const { firstCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
|
assert.equal(firstCommand('` # fill in`').reason, 'not-a-command');
|
|
assert.equal(firstCommand('`{exact command}`').reason, 'placeholder', 'the control');
|
|
});
|
|
|
|
test('review 630 r8 / criteria-runner.mjs:175: a # inside a word is kept by dropComment and refused by the allowlist, and the text says so', async () => {
|
|
const { firstCommand, allowedCommand } = await import('../../lib/verification/criteria-runner.mjs');
|
|
assert.equal(firstCommand('`node --test tests/a#b.test.mjs`').command, 'node --test tests/a#b.test.mjs');
|
|
assert.equal(allowedCommand('node --test tests/a#b.test.mjs', { cwd: ROOT }).allowed, false);
|
|
const doc = /\/\*\*\n((?: \*[^\n]*\n)+) \*\/\nexport function dropComment/.exec(read('lib/verification/criteria-runner.mjs'));
|
|
assert.ok(doc, 'dropComment has its doc comment');
|
|
assert.match(doc[1].replace(/\n \* ?/g, ' '), /a#b\.test\.mjs`?[^.]*\bkept\b[^.]*\ballowlist\b[^.]*\brefuses\b/,
|
|
'the doc says the allowlist still refuses the word');
|
|
});
|
|
|
|
test('review 630 r8 / criteria-runner.mjs:890: no CLI compares process.argv[1] itself, and the three that did work through a symlinked path with a space', () => {
|
|
const files = spawnSync('git', ['ls-files', 'lib', 'scripts', 'hooks'], { cwd: ROOT, encoding: 'utf8' }).stdout.split('\n').filter((p) => p.endsWith('.mjs'));
|
|
assert.deepEqual(files.filter((p) => p !== 'lib/util/is-main.mjs' && /process\.argv\[1\]/.test(read(p))), [],
|
|
'only lib/util/is-main.mjs reads process.argv[1]');
|
|
const base = mkdtempSync(join(tmpdir(), 'r630 r8 main '));
|
|
const link = join(base, 'voyage link');
|
|
try {
|
|
symlinkSync(ROOT, link);
|
|
for (const cli of ['lib/verification/criteria-runner.mjs', 'scripts/comparison-gate.mjs', 'scripts/end-state-gate.mjs']) {
|
|
const r = spawnSync(process.execPath, [join(link, cli), '--no-such-flag'], { encoding: 'utf8', input: '', timeout: 30_000 });
|
|
assert.equal(r.status, 2, `${cli} through "${link}": ${r.stdout}${r.stderr}`);
|
|
assert.match(r.stderr, /unknown argument --no-such-flag/);
|
|
}
|
|
} finally { fs.rmSync(link, { force: true }); rmSync(base, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('review 630 r8 / plan-schema.mjs:353 + :322: a description beside a backticked path names no file; a build file in its own entry does', () => {
|
|
const step = (files) => `- **Files:** ${files}\n- **Changes:** x\n`;
|
|
// A description after a dash, or a word inside a sentence, is no path (code-review, code-correctness).
|
|
for (const files of ['`tests/lib/x.test.mjs` — mocks fs.readFileSync', '`tests/foo.test.mjs` — covers lib/foo.mjs',
|
|
'`tests/a.test.mjs`, runs on Node.js 22.x', '`tests/a.test.mjs` — exercises the parser in index.js',
|
|
'`tests/a.test.mjs`, no change to package.json', '`tests/a.test.mjs` - asserts JSON.parse behaviour',
|
|
'`tests/a.test.mjs`: covers src/a.mjs and Makefile']) {
|
|
assert.deepEqual(stepTestsOnly(step(files)), { testsOnly: true, files: ['tests/a.test.mjs', 'tests/foo.test.mjs', 'tests/lib/x.test.mjs'].filter((t) => files.startsWith(`\`${t}\``)) }, files);
|
|
}
|
|
// A file in its own entry, after "and", or a path with a slash before any description, still counts (round 7).
|
|
for (const [files, paths] of [
|
|
['`tests/a.test.mjs`, makefile', ['tests/a.test.mjs', 'makefile']],
|
|
['`tests/a.test.mjs`, justfile', ['tests/a.test.mjs', 'justfile']],
|
|
['`tests/a.test.mjs`, package.json', ['tests/a.test.mjs', 'package.json']],
|
|
['`tests/a.test.mjs` and package.json', ['tests/a.test.mjs', 'package.json']],
|
|
['`tests/a.test.mjs`, the CLI entry src/cli.mjs — reads argv', ['tests/a.test.mjs', 'src/cli.mjs']],
|
|
]) {
|
|
assert.deepEqual(stepTestsOnly(step(files)), { testsOnly: false, files: paths }, files);
|
|
}
|
|
});
|
|
|
|
test('review 630 r8 / done-line.mjs:61 + test-lock.mjs:183: a stopped command is read by stoppedBy, the one reading', async () => {
|
|
const { writeDoneLine } = await import('../../lib/verification/done-line.mjs');
|
|
const repo = gitRepo('r630-r8-done-');
|
|
try {
|
|
writeFileSync(join(repo, 'README.md'), 'x\n');
|
|
commitAll(repo, 'base');
|
|
mkdirSync(join(repo, 'tests'));
|
|
writeFileSync(join(repo, 'tests', 'hang.test.mjs'), "import { test } from 'node:test';\ntest('hangs', () => new Promise((r) => setTimeout(r, 6000)));\n");
|
|
const r = writeDoneLine({ cwd: repo, step: 1, cmd: 'node --test tests/hang.test.mjs', report: join(repo, 'report.md'), timeoutMs: 2000 });
|
|
assert.equal(r.status, 'not-run', JSON.stringify(r));
|
|
assert.match(r.reason, /killed .*after the 2 s timeout/, r.reason);
|
|
} finally { rmSync(repo, { recursive: true, force: true }); }
|
|
for (const rel of ['lib/verification/done-line.mjs', 'lib/util/test-lock.mjs', 'lib/verification/red-proof.mjs']) {
|
|
const src = read(rel);
|
|
assert.match(src, /\bstoppedBy\(/, `${rel} reads a stop with stoppedBy`);
|
|
assert.doesNotMatch(src, /r\.error \|\| r\.status === null/, `${rel} does not read the spawn result by hand`);
|
|
}
|
|
});
|
|
|
|
test('review 630 r8 / red-proof.mjs:101: a run reports one reading of a stop, not a second spawnError beside it', async () => {
|
|
const { proveRed } = await import('../../lib/verification/red-proof.mjs');
|
|
const repo = gitRepo('r630-r8-spawn-');
|
|
try {
|
|
writeFileSync(join(repo, 'README.md'), 'x\n');
|
|
const base = commitAll(repo, 'base');
|
|
mkdirSync(join(repo, 'tests'));
|
|
writeFileSync(join(repo, 'tests', 'a.test.mjs'), 'x\n');
|
|
// `jest` is on the allowlist and not installed where this suite runs.
|
|
const r = proveRed({ cwd: repo, base, cmd: 'jest tests/a.test.mjs', tests: ['tests/a.test.mjs'], redOnly: true });
|
|
assert.equal(r.status, 'not-run', JSON.stringify(r));
|
|
assert.match(r.reason, /could not start/, r.reason);
|
|
const runs = [r.base, r.result].filter(Boolean);
|
|
assert.ok(runs.length > 0, JSON.stringify(r));
|
|
for (const run of runs) assert.ok(!('spawnError' in run), `no spawnError beside stopped: ${JSON.stringify(run)}`);
|
|
} finally { rmSync(repo, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('review 630 r8 / findings-schema.mjs:46: the block reader and the verdict gate agree on what a fence is', async () => {
|
|
const { extractFindingsBlock } = await import('../../lib/review/findings-schema.mjs');
|
|
const json = '{\n "findings": []\n}';
|
|
// Markdown: a closing fence may be longer than the opening one.
|
|
assert.equal(extractFindingsBlock(`x\n\n\`\`\`json\n${json}\n\`\`\`\`\n`), `${json}\n`, 'a ```` closer ends the block');
|
|
// Not a fence in Markdown, and not in the verdict gate either: three backticks
|
|
// after text on the same line.
|
|
assert.equal(extractFindingsBlock(`\`\`\`json\n${json}\`\`\`\n`), null, 'a closer after JSON on its line');
|
|
assert.equal(extractFindingsBlock(`see \`\`\`json\n${json}\n\`\`\`\n`), null, 'an opener after prose on its line');
|
|
const { decideVerdict } = await import('../../lib/review/verdict-gate.mjs');
|
|
const fm = ['---', 'type: trekreview', 'review_version: "1.1"', 'created: 2026-10-11', 'task: "t"', 'slug: s',
|
|
'project_dir: p/', 'brief_path: p/brief.md', `scope_sha_start: ${'b'.repeat(40)}`, `scope_sha_end: ${'c'.repeat(40)}`,
|
|
'reviewed_files_count: 1', 'verdict: WARN', 'findings: []', '---', ''].join('\n');
|
|
const payload = JSON.stringify({ verdict: 'WARN', counts: { BLOCKER: 0, MAJOR: 0, MINOR: 0, SUGGESTION: 0 }, allow_blocked_by: [], findings: [] }, null, 2);
|
|
for (const closer of ['```', '````']) {
|
|
const v = decideVerdict(`${fm}\n# R\n\n## Executive Summary\n\nx\n\n## Coverage\n\nx\n\n## Declined to Judge\n\nNone.\n\nTrimmed: 0\n\n` +
|
|
`## Remediation Summary\n\nx\n\n\`\`\`json\n${payload}\n${closer}\n`);
|
|
assert.equal(v.exit, 0, `closer ${closer}: ${v.reason}`);
|
|
}
|
|
});
|
|
|
|
test('review 630 r8 / review-validator.mjs:36 + synthesis-digest-schema.mjs:59: three backticks in a JSON string do not end the block there either', async () => {
|
|
const { validateReviewContent } = await import('../../lib/validators/review-validator.mjs');
|
|
const { extractDigestBlock } = await import('../../lib/plan/synthesis-digest-schema.mjs');
|
|
const id = 'a'.repeat(40);
|
|
const payload = { verdict: 'BLOCK', counts: { BLOCKER: 1, MAJOR: 0, MINOR: 0, SUGGESTION: 0 }, allow_blocked_by: [],
|
|
findings: [{ id, severity: 'BLOCKER', rule_key: 'MISSING_TEST', file: 'a.mjs', line: 1, title: 'x',
|
|
detail: 'write the check as ```bash\nnpm test\n``` and run it', recommended_action: 'Add a test.' }] };
|
|
const review = ['---', 'type: trekreview', 'review_version: "1.1"', 'created: 2026-10-11', 'task: "t"', 'slug: s',
|
|
'project_dir: p/', 'brief_path: p/brief.md', 'scope_sha_start: ' + 'b'.repeat(40), 'scope_sha_end: ' + 'c'.repeat(40),
|
|
'reviewed_files_count: 1', 'verdict: BLOCK', 'findings:', ` - ${id}`, '---', '', '# Review', '',
|
|
'## Executive Summary', '', 'x', '', '## Coverage', '', 'x', '', '## Declined to Judge', '', 'None.', '', 'Trimmed: 0', '',
|
|
'## Remediation Summary', '', 'x', '', '```json', JSON.stringify(payload, null, 2), '```', ''].join('\n');
|
|
const r = validateReviewContent(review);
|
|
assert.ok([...r.errors, ...r.warnings].some((i) => i.code === 'REVIEW_MISSING_REMEDIATION_STEP'),
|
|
`a BLOCKER with no ### Step R<n> is found although its detail holds a fence: ${JSON.stringify(r.errors.map((e) => e.code))}`);
|
|
const digest = '```json\n{"note": "a ```bash fence"}\n```\n';
|
|
assert.equal(extractDigestBlock(digest), '{"note": "a ```bash fence"}\n');
|
|
});
|