feat(verification): K6 and the red proof read a 1.9 plan by its grammar [skip-docs]

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-12 16:29:58 +02:00
commit d14a49b92d
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
3 changed files with 90 additions and 6 deletions

View file

@ -12,6 +12,13 @@ import * as schema from '../../lib/parsers/plan-schema.mjs';
import { validatePlanContent } from '../../lib/validators/plan-validator.mjs';
import { gatePlan, recordVerdict, planHash } from '../../lib/validators/plan-gate.mjs';
import { buildPlan } from '../helpers/plan-v18-builder.mjs';
import { plannedPaths } from '../../lib/verification/plan-scope-sweep.mjs';
import { childEnv } from '../helpers/child-env.mjs';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
const errCodes = (r) => r.errors.map((e) => e.code);
const warnCodes = (r) => r.warnings.map((w) => w.code);
@ -152,3 +159,67 @@ test('below 1.9: a plan whose Files are all backticked gets no new warning', ()
assert.ok(errCodes(v).includes('PLAN_STEP_PATH_NOT_FIRST'), errCodes(v).join(','));
assert.equal(v.warnings.find((w) => w.code === 'PLAN_STEP_PATH_UNTICKED'), undefined);
});
// ---- Step 6: K6 and the red proof read a 1.9 plan by the grammar ----
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
const RED_PROOF = join(ROOT, 'lib', 'verification', 'red-proof.mjs');
const COVERS = '`tests/a.test.mjs` - covers `src/parse.mjs`';
// Step 1 names only its test, and the description mentions the source it covers.
// The manifest names the test alone, so it adds nothing K6 would not plan anyway.
const coversPlan = (version) => buildPlan({ version, steps: [
{ filesRaw: COVERS, test: 'tests/a.test.mjs', manifestPaths: ['tests/a.test.mjs'] },
{ endToEnd: 'bash tests/e2e.sh' },
] });
test('1.9: K6 plans the files a 1.9 Files: entry names, never a path its description mentions', () => {
const text = coversPlan('1.9');
assert.ok(text.includes(`- **Files:** ${COVERS}\n`), 'the builder writes the raw Files value');
assert.deepEqual(plannedPaths(text), ['src/mod2.mjs', 'tests/a.test.mjs', 'tests/mod2.test.mjs']);
});
test('1.9: K6 plans the files a 1.9 Test first File: line names, never its description', () => {
const text = buildPlan({ version: '1.9', steps: [
{ filesRaw: '`tests/a.test.mjs`', testFirstRaw: ` - File: ${COVERS}`, test: 'tests/a.test.mjs', manifestPaths: ['tests/a.test.mjs'] },
{ endToEnd: 'bash tests/e2e.sh' },
] });
assert.ok(text.includes(` - File: ${COVERS}\n`), 'the builder writes the raw File line');
assert.deepEqual(plannedPaths(text), ['src/mod2.mjs', 'tests/a.test.mjs', 'tests/mod2.test.mjs']);
});
test('below 1.9: K6 plans what it planned before - the same input plans both paths', () => {
assert.deepEqual(plannedPaths(coversPlan('1.8')), ['src/mod2.mjs', 'src/parse.mjs', 'tests/a.test.mjs', 'tests/mod2.test.mjs']);
const tf = buildPlan({ version: '1.8', steps: [
{ filesRaw: '`tests/a.test.mjs`', testFirstRaw: ` - File: ${COVERS}`, test: 'tests/a.test.mjs', manifestPaths: ['tests/a.test.mjs'] },
{ endToEnd: 'bash tests/e2e.sh' },
] });
assert.deepEqual(plannedPaths(tf), ['src/mod2.mjs', 'src/parse.mjs', 'tests/a.test.mjs', 'tests/mod2.test.mjs']);
});
test('1.9: the red proof\'s tests_only reads Files by the grammar (CLI, temp git repo); 1.8 reads it as before', () => {
const repo = mkdtempSync(join(tmpdir(), 'plan19-redproof-'));
const git = (...args) => spawnSync('git', args, { cwd: repo, encoding: 'utf8', env: childEnv() });
try {
for (const args of [['init', '-q'], ['config', 'user.email', 't@example.invalid'], ['config', 'user.name', 't'], ['config', 'commit.gpgsign', 'false']]) {
assert.equal(git(...args).status, 0);
}
writeFileSync(join(repo, 'README.md'), 'x\n');
assert.equal(git('add', '-A').status, 0);
assert.equal(git('commit', '-q', '-m', 'base').status, 0);
const base = git('rev-parse', 'HEAD').stdout.trim();
// The helper wrote the step's test alone; it passes at the start point.
mkdirSync(join(repo, 'tests'));
writeFileSync(join(repo, 'tests', 'a.test.mjs'), "import { test } from 'node:test';\ntest('a', () => {});\n");
mkdirSync(join(repo, 'plans'));
for (const [version, want] of [['1.9', true], ['1.8', false]]) {
const plan = join(repo, 'plans', `plan-${version}.md`);
writeFileSync(plan, coversPlan(version));
const r = spawnSync('node', [RED_PROOF, '--red-only', '--base', base, '--cmd', 'node --test tests/a.test.mjs',
'--tests', 'tests/a.test.mjs', '--plan', plan, '--step', '1', '--json'], { cwd: repo, encoding: 'utf8', env: childEnv() });
assert.equal(r.status, 1, `${version}: ${r.stdout}${r.stderr}`);
const out = JSON.parse(r.stdout);
assert.equal(out.reason, 'not-red-at-base', JSON.stringify(out));
assert.equal(out.tests_only, want, `plan_version ${version}: ${JSON.stringify(out.tests_only_note ?? null)}`);
}
} finally { rmSync(repo, { recursive: true, force: true }); }
});