The operator window's step [a] called 40-validate-standalone.sh directly (strict exit-code), so it STOPped on the first target carrying pre-existing in-repo test red — voyage (2 doc-consistency drifts re phase_models/phase_signals, content moved to docs/operations.md) and ai-psychosis (1). But the migration's ratified contract, the one the Step-11 dry-run validated as PASS 11/11, is 'introduce no regression': pre-existing in-repo red is the plugin's own concern, not a migration regression. The window enforced a STRICTER gate than the contract the dry-run signed off. Fix: new 41-validate-or-regression.sh — the single per-target gate the window calls in [a]. It runs 40 strict, then on failure passes iff the standalone failing-test NAME set is a SUBSET of the live in-repo set (the exact decision 99-dryrun.sh makes), reusing sc2-regression.sh. A genuine extraction-introduced regression still STOPs the window; a structure-validator fail and the config-audit gate stay strict. Single-source the failing-name capture: extract capture_fails into capture-fails.sh (mirrors the sc2-regression.sh extraction) so the live gate and the dry-run agree on what 'failing' means; 99-dryrun.sh now delegates to it (behaviour identical). Verified end-to-end on the real extracts: 40 strict fails voyage+ai-psychosis while 41 passes them 'N pre-existing, regression-relative'; clean targets (llm-security, graceful-handoff) still pass via the strict path. New hermetic tests: capture-fails 3/3, 41 6/6 (strict-pass, regression-relative-pass, genuine-regression-fail, structure-not-eligible, gate pass/fail). RUNBOOK per-repo step updated to 41. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
73 lines
3.7 KiB
JavaScript
73 lines
3.7 KiB
JavaScript
// Coverage for capture-fails.sh — the failing-test-NAME capture extracted from 99-dryrun.sh so a SINGLE
|
|
// source feeds both the dry-run rehearsal and the live operator-window gate (41-validate-or-regression.sh).
|
|
// Proves it reports exactly the failing names (sorted, deduped across files) and stays exit-0 on an all-pass
|
|
// suite — a no-match grep must NOT leak a non-zero status up to the caller (which would masquerade as a
|
|
// failed capture). Drives the real script against tiny node:test fixtures — hermetic, no migration state.
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
const CAP = path.join(here, 'capture-fails.sh');
|
|
const CMD = "node --test 'tests/**/*.test.mjs'";
|
|
|
|
// One node:test file = imports ONCE + a test() per entry. entry: { name, pass }. (Concatenating
|
|
// per-name single-file strings would duplicate `import test` → SyntaxError → a file-level not-ok, masking
|
|
// the per-test capture under test.)
|
|
const suiteFile = (entries) =>
|
|
"import test from 'node:test';\nimport assert from 'node:assert/strict';\n" +
|
|
entries.map((e) => `test(${JSON.stringify(e.name)}, () => assert.equal(1, ${e.pass ? 1 : 2}));`).join('\n') +
|
|
'\n';
|
|
|
|
function sandbox(files) {
|
|
const dir = mkdtempSync(path.join(os.tmpdir(), 'capfail-'));
|
|
for (const [rel, content] of Object.entries(files)) {
|
|
const p = path.join(dir, rel);
|
|
mkdirSync(path.dirname(p), { recursive: true });
|
|
writeFileSync(p, content);
|
|
}
|
|
return dir;
|
|
}
|
|
|
|
function cap(dir) {
|
|
// Strip NODE_TEST_CONTEXT: capture-fails spawns `node --test`, which misbehaves (emits no TAP) when it
|
|
// inherits the harness's nested-test context (Step-6 env-clean gotcha). The real operator window runs as
|
|
// plain bash, never nested under node:test, so this only matters inside this harness.
|
|
const env = { ...process.env };
|
|
delete env.NODE_TEST_CONTEXT;
|
|
return spawnSync('bash', [CAP, dir, CMD], { encoding: 'utf8', env });
|
|
}
|
|
|
|
test('capture-fails reports the failing test name (and only it), exit 0', () => {
|
|
const dir = sandbox({ 'tests/a.test.mjs': suiteFile([{ name: 'good', pass: true }, { name: 'bad apple', pass: false }]) });
|
|
try {
|
|
const r = cap(dir);
|
|
assert.equal(r.status, 0, `capture must exit 0 even with failures: ${r.stderr}`);
|
|
assert.equal(r.stdout.trim(), 'bad apple', `only the failing name expected, got: ${JSON.stringify(r.stdout)}`);
|
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('capture-fails on an all-pass suite → empty output, exit 0 (no-match grep must not leak non-zero)', () => {
|
|
const dir = sandbox({ 'tests/a.test.mjs': suiteFile([{ name: 'good', pass: true }, { name: 'also good', pass: true }]) });
|
|
try {
|
|
const r = cap(dir);
|
|
assert.equal(r.status, 0, `all-pass must exit 0: ${r.stderr}`);
|
|
assert.equal(r.stdout.trim(), '', `no failing names expected, got: ${JSON.stringify(r.stdout)}`);
|
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('capture-fails sorts + dedups failing names across files', () => {
|
|
const dir = sandbox({
|
|
'tests/a.test.mjs': suiteFile([{ name: 'zeta', pass: false }, { name: 'alpha', pass: false }]),
|
|
'tests/b.test.mjs': suiteFile([{ name: 'alpha', pass: false }]), // duplicate name across files collapses to one
|
|
});
|
|
try {
|
|
const r = cap(dir);
|
|
assert.equal(r.status, 0, r.stderr);
|
|
assert.equal(r.stdout.trim(), 'alpha\nzeta', `sorted+deduped expected, got: ${JSON.stringify(r.stdout)}`);
|
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
|
});
|