fix(tests): the control-run test leaves headroom over a slow start
Review finding 703d87e1 (BLOCKER); the timeout and hang live in tests/helpers/timing.mjs as CONTROL_RUN. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
4d5c927c5d
commit
e142556543
3 changed files with 51 additions and 5 deletions
|
|
@ -23,3 +23,15 @@ export async function fastestWithin(limitMs, measure, tries = 3) {
|
|||
const fastest = Math.min(...runs);
|
||||
return { ok: fastest < limitMs, fastest, runs };
|
||||
}
|
||||
|
||||
// The red-proof control-run test (review 630 r9, brief note 3) reads these.
|
||||
// Red proof gives its three runs one timeout, and the test needs the result and
|
||||
// start-point runs to finish under it on a loaded machine: the criteria runner
|
||||
// saw the start point killed at 2 s under full-suite load (review 703d87e1).
|
||||
// The timeout is raised on purpose and is not a loosened assertion: the test
|
||||
// checks which run red proof stopped, not how fast a run is. `slowStartMs`
|
||||
// simulates a slow cold start, above the old 2 s; `hangMs` outlasts the timeout
|
||||
// by 30 s with a real timer. This is the raise of the test's own threshold
|
||||
// (2000 -> 15000 ms, 6000 -> 45000 ms) that the floor guard may report as
|
||||
// loosened-threshold: plan step 1 of round 3 asks for it.
|
||||
export const CONTROL_RUN = Object.freeze({ timeoutMs: 15_000, hangMs: 45_000, slowStartMs: 2_500 });
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import { validateFindings } from '../../lib/review/findings-schema.mjs';
|
|||
import { lintSkill } from '../../lib/skills/skill-lint.mjs';
|
||||
import * as routing from '../../lib/skills/routing.mjs';
|
||||
import { childEnv } from '../helpers/child-env.mjs';
|
||||
import * as timing from '../helpers/timing.mjs';
|
||||
|
||||
const awaitImportSync = () => routing;
|
||||
|
||||
|
|
@ -1159,7 +1160,17 @@ test('review 630 r9 / 7b3750e9 plan-schema.mjs:372: a plain path that ends with
|
|||
assert.deepEqual(stepTestsOnly(step('`tests/a.test.mjs`, note: Makefile and package.json')).files, ['tests/a.test.mjs', 'package.json']);
|
||||
});
|
||||
|
||||
test('review 630 r9 / brief note 3: red proof reports a control run it stopped as not-run, in the control run', async () => {
|
||||
// Review 703d87e1: under full-suite load the criteria runner saw this test's
|
||||
// start-point run killed at the old 2 s timeout ("at the start point"), not the
|
||||
// control run. Its numbers live in tests/helpers/timing.mjs (CONTROL_RUN): the
|
||||
// timeout leaves headroom over a slow start, the hang outlasts it. Every run
|
||||
// but the control starts slowly on purpose (slowStartMs), so a timeout without
|
||||
// headroom fails here on demand, not only under load. BEFORE_703D87E1 is the
|
||||
// record of the numbers before the fix, taken only when the export is absent.
|
||||
const BEFORE_703D87E1 = Object.freeze({ timeoutMs: 2000, hangMs: 6000, slowStartMs: 2500 });
|
||||
|
||||
test('review 630 r9 / brief note 3 + 703d87e1: red proof reports a control run it stopped as not-run, in the control run, when every other run starts slowly', async () => {
|
||||
const c = timing.CONTROL_RUN ?? BEFORE_703D87E1;
|
||||
const { proveRed } = await import('../../lib/verification/red-proof.mjs');
|
||||
const repo = gitRepo('r630-r9-control-');
|
||||
try {
|
||||
|
|
@ -1168,18 +1179,21 @@ test('review 630 r9 / brief note 3: red proof reports a control run it stopped a
|
|||
mkdirSync(join(repo, 'tests'));
|
||||
mkdirSync(join(repo, 'src'));
|
||||
// Red at the start point (no src/impl.mjs), green in the working tree, and
|
||||
// it hangs only in the control: the start-point copy with the change laid on.
|
||||
// it hangs only in the control: the start-point copy with the change laid
|
||||
// on. Every other run waits slowStartMs before its one test.
|
||||
writeFileSync(join(repo, 'tests', 'a.test.mjs'), [
|
||||
"import { test } from 'node:test';",
|
||||
"import { existsSync } from 'node:fs';",
|
||||
"const control = process.cwd().includes('voyage-red-proof-') && existsSync('src/impl.mjs');",
|
||||
`if (!control) await new Promise((r) => setTimeout(r, ${c.slowStartMs}));`,
|
||||
"test('impl', async () => {",
|
||||
" if (!existsSync('src/impl.mjs')) throw new Error('no impl');",
|
||||
" if (process.cwd().includes('voyage-red-proof-')) await new Promise((r) => setTimeout(r, 6000));",
|
||||
` if (control) await new Promise((r) => setTimeout(r, ${c.hangMs}));`,
|
||||
'});', ''].join('\n'));
|
||||
writeFileSync(join(repo, 'src', 'impl.mjs'), 'export const x = 1;\n');
|
||||
const r = proveRed({ cwd: repo, base, cmd: 'node --test tests/a.test.mjs', tests: ['tests/a.test.mjs'], timeoutMs: 2000 });
|
||||
const r = proveRed({ cwd: repo, base, cmd: 'node --test tests/a.test.mjs', tests: ['tests/a.test.mjs'], timeoutMs: c.timeoutMs });
|
||||
assert.equal(r.status, 'not-run', JSON.stringify(r));
|
||||
assert.match(r.reason, /^in the control run: the command was killed .*after the 2 s timeout/, r.reason);
|
||||
assert.match(r.reason, new RegExp(`^in the control run: the command was killed .*after the ${c.timeoutMs / 1000} s timeout`), r.reason);
|
||||
assert.equal(r.base.exit, 1, 'red at the start point');
|
||||
assert.equal(r.result.exit, 0, 'green at the result');
|
||||
assert.ok(r.control, 'the control run is reported');
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { fastestWithin } from '../helpers/timing.mjs';
|
||||
import * as timing from '../helpers/timing.mjs';
|
||||
import { HOOK_TIMEOUT_MS, runHookWithEnv } from '../helpers/hook-helper.mjs';
|
||||
|
||||
/** A measure that returns the given durations in turn (no real waiting). */
|
||||
|
|
@ -61,3 +62,22 @@ test('hook helper: the kill limit is far above 5 s by default, and a call can se
|
|||
assert.equal(done.stdout, 'done');
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
// The red-proof control-run test (review 630 r9, brief note 3, in
|
||||
// tests/lib/review-630.test.mjs) reads its timeout, hang and simulated slow
|
||||
// start from tests/helpers/timing.mjs (CONTROL_RUN, review 703d87e1): the
|
||||
// criteria runner saw its start point killed at 2 s under full-suite load.
|
||||
// BEFORE_703D87E1 is the record of the numbers before the fix, taken only when
|
||||
// the export is absent.
|
||||
const BEFORE_703D87E1 = Object.freeze({ timeoutMs: 2000, hangMs: 6000, slowStartMs: 2500 });
|
||||
|
||||
test('red proof control run (703d87e1): the timeout leaves headroom over a slow start and the hang outlasts it', () => {
|
||||
const c = timing.CONTROL_RUN ?? BEFORE_703D87E1;
|
||||
const seen = JSON.stringify(c);
|
||||
assert.ok(c.slowStartMs > 2000, `the simulated slow start is above the old 2 s timeout: ${seen}`);
|
||||
assert.ok(c.timeoutMs >= 15000, `the timeout leaves headroom over a loaded machine: ${seen}`);
|
||||
assert.equal(c.timeoutMs % 1000, 0, `a whole number of seconds (the reason prints Math.round(timeoutMs / 1000)): ${seen}`);
|
||||
assert.ok(c.timeoutMs >= 6 * c.slowStartMs, `the timeout is at least 6x the simulated slow start: ${seen}`);
|
||||
assert.ok(c.hangMs >= c.timeoutMs + 30000, `the hang outlasts the timeout by 30 s: ${seen}`);
|
||||
assert.ok(c.hangMs < 2 ** 31 - 1, `the hang fits a real timer (below 2^31-1 ms): ${seen}`);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue