feat(verification): K6 and the red proof read a 1.9 plan by its grammar [skip-docs]
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
fc157c3ad5
commit
d14a49b92d
3 changed files with 90 additions and 6 deletions
|
|
@ -12,6 +12,13 @@ import * as schema from '../../lib/parsers/plan-schema.mjs';
|
|||
import { validatePlanContent } from '../../lib/validators/plan-validator.mjs';
|
||||
import { gatePlan, recordVerdict, planHash } from '../../lib/validators/plan-gate.mjs';
|
||||
import { buildPlan } from '../helpers/plan-v18-builder.mjs';
|
||||
import { plannedPaths } from '../../lib/verification/plan-scope-sweep.mjs';
|
||||
import { childEnv } from '../helpers/child-env.mjs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const errCodes = (r) => r.errors.map((e) => e.code);
|
||||
const warnCodes = (r) => r.warnings.map((w) => w.code);
|
||||
|
|
@ -152,3 +159,67 @@ test('below 1.9: a plan whose Files are all backticked gets no new warning', ()
|
|||
assert.ok(errCodes(v).includes('PLAN_STEP_PATH_NOT_FIRST'), errCodes(v).join(','));
|
||||
assert.equal(v.warnings.find((w) => w.code === 'PLAN_STEP_PATH_UNTICKED'), undefined);
|
||||
});
|
||||
|
||||
// ---- Step 6: K6 and the red proof read a 1.9 plan by the grammar ----
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const RED_PROOF = join(ROOT, 'lib', 'verification', 'red-proof.mjs');
|
||||
const COVERS = '`tests/a.test.mjs` - covers `src/parse.mjs`';
|
||||
// Step 1 names only its test, and the description mentions the source it covers.
|
||||
// The manifest names the test alone, so it adds nothing K6 would not plan anyway.
|
||||
const coversPlan = (version) => buildPlan({ version, steps: [
|
||||
{ filesRaw: COVERS, test: 'tests/a.test.mjs', manifestPaths: ['tests/a.test.mjs'] },
|
||||
{ endToEnd: 'bash tests/e2e.sh' },
|
||||
] });
|
||||
|
||||
test('1.9: K6 plans the files a 1.9 Files: entry names, never a path its description mentions', () => {
|
||||
const text = coversPlan('1.9');
|
||||
assert.ok(text.includes(`- **Files:** ${COVERS}\n`), 'the builder writes the raw Files value');
|
||||
assert.deepEqual(plannedPaths(text), ['src/mod2.mjs', 'tests/a.test.mjs', 'tests/mod2.test.mjs']);
|
||||
});
|
||||
|
||||
test('1.9: K6 plans the files a 1.9 Test first File: line names, never its description', () => {
|
||||
const text = buildPlan({ version: '1.9', steps: [
|
||||
{ filesRaw: '`tests/a.test.mjs`', testFirstRaw: ` - File: ${COVERS}`, test: 'tests/a.test.mjs', manifestPaths: ['tests/a.test.mjs'] },
|
||||
{ endToEnd: 'bash tests/e2e.sh' },
|
||||
] });
|
||||
assert.ok(text.includes(` - File: ${COVERS}\n`), 'the builder writes the raw File line');
|
||||
assert.deepEqual(plannedPaths(text), ['src/mod2.mjs', 'tests/a.test.mjs', 'tests/mod2.test.mjs']);
|
||||
});
|
||||
|
||||
test('below 1.9: K6 plans what it planned before - the same input plans both paths', () => {
|
||||
assert.deepEqual(plannedPaths(coversPlan('1.8')), ['src/mod2.mjs', 'src/parse.mjs', 'tests/a.test.mjs', 'tests/mod2.test.mjs']);
|
||||
const tf = buildPlan({ version: '1.8', steps: [
|
||||
{ filesRaw: '`tests/a.test.mjs`', testFirstRaw: ` - File: ${COVERS}`, test: 'tests/a.test.mjs', manifestPaths: ['tests/a.test.mjs'] },
|
||||
{ endToEnd: 'bash tests/e2e.sh' },
|
||||
] });
|
||||
assert.deepEqual(plannedPaths(tf), ['src/mod2.mjs', 'src/parse.mjs', 'tests/a.test.mjs', 'tests/mod2.test.mjs']);
|
||||
});
|
||||
|
||||
test('1.9: the red proof\'s tests_only reads Files by the grammar (CLI, temp git repo); 1.8 reads it as before', () => {
|
||||
const repo = mkdtempSync(join(tmpdir(), 'plan19-redproof-'));
|
||||
const git = (...args) => spawnSync('git', args, { cwd: repo, encoding: 'utf8', env: childEnv() });
|
||||
try {
|
||||
for (const args of [['init', '-q'], ['config', 'user.email', 't@example.invalid'], ['config', 'user.name', 't'], ['config', 'commit.gpgsign', 'false']]) {
|
||||
assert.equal(git(...args).status, 0);
|
||||
}
|
||||
writeFileSync(join(repo, 'README.md'), 'x\n');
|
||||
assert.equal(git('add', '-A').status, 0);
|
||||
assert.equal(git('commit', '-q', '-m', 'base').status, 0);
|
||||
const base = git('rev-parse', 'HEAD').stdout.trim();
|
||||
// The helper wrote the step's test alone; it passes at the start point.
|
||||
mkdirSync(join(repo, 'tests'));
|
||||
writeFileSync(join(repo, 'tests', 'a.test.mjs'), "import { test } from 'node:test';\ntest('a', () => {});\n");
|
||||
mkdirSync(join(repo, 'plans'));
|
||||
for (const [version, want] of [['1.9', true], ['1.8', false]]) {
|
||||
const plan = join(repo, 'plans', `plan-${version}.md`);
|
||||
writeFileSync(plan, coversPlan(version));
|
||||
const r = spawnSync('node', [RED_PROOF, '--red-only', '--base', base, '--cmd', 'node --test tests/a.test.mjs',
|
||||
'--tests', 'tests/a.test.mjs', '--plan', plan, '--step', '1', '--json'], { cwd: repo, encoding: 'utf8', env: childEnv() });
|
||||
assert.equal(r.status, 1, `${version}: ${r.stdout}${r.stderr}`);
|
||||
const out = JSON.parse(r.stdout);
|
||||
assert.equal(out.reason, 'not-red-at-base', JSON.stringify(out));
|
||||
assert.equal(out.tests_only, want, `plan_version ${version}: ${JSON.stringify(out.tests_only_note ?? null)}`);
|
||||
}
|
||||
} finally { rmSync(repo, { recursive: true, force: true }); }
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue