diff --git a/docs/marketplace-polyrepo-migration/migration/41-validate-or-regression.sh b/docs/marketplace-polyrepo-migration/migration/41-validate-or-regression.sh new file mode 100644 index 0000000..47b7124 --- /dev/null +++ b/docs/marketplace-polyrepo-migration/migration/41-validate-or-regression.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Step 6b — per-target SC2/SC7 gate with the migration's REGRESSION-RELATIVE contract baked in. +# +# WHY THIS EXISTS: 40-validate-standalone.sh is STRICT — any standalone test failure exits non-zero. The +# migration's RATIFIED contract — the one the Step-11 dry-run (99-dryrun.sh) validated and signed off as +# "PASS 11/11" — is weaker and correct: a target passes iff the extraction introduces NO NEW failure. +# Pre-existing in-repo red (voyage's 2 doc-consistency drifts re phase_models/phase_signals; ai-psychosis's +# 1) is the PLUGIN's own concern, not a migration regression. The operator window (run-operator-window.sh +# step [a]) must enforce THAT contract, not a stricter one — otherwise it STOPs on the first target carrying +# pre-existing red even though the dry-run blessed it (the bug this script fixes). This is the single +# per-target gate the window calls; it mirrors 99-dryrun.sh's SC2 decision exactly, reusing capture-fails.sh +# (the failing-name capture) + sc2-regression.sh (the subset decision). NULL push (D8): read-only validation. +# +# CONTRACT: +# - sc2_gate target (config-audit): the dedicated gate is deterministic — run it STRICT (PASS/FAIL). +# - else: run 40-validate-standalone.sh STRICT. PASS => exit 0. +# On FAIL, only a `node --test` suite is regression-eligible: its standalone failing-NAME set must be +# a SUBSET of the live in-repo failing-NAME set (regression-relative PASS, "PASS (N pre-existing)"). +# A structure-validator (validate-plugin*.sh / bash -c) FAIL is a genuine structural defect — NOT +# softened. A genuine regression (standalone-only failure) or a missing capture => exit non-zero. +# Escalate, never mask: any real regression or structural failure exits non-zero so the window STOPs. +# +# Usage: 41-validate-or-regression.sh +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MAP="$SCRIPT_DIR/plugin-map.json" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +WORK="${WORK:-/tmp/polyrepo-migration}" + +[ -f "$MAP" ] || { echo "plugin-map.json missing at $MAP" >&2; exit 2; } +key="${1:?usage: 41-validate-or-regression.sh }" + +mapget() { python3 -c "import json;print(json.load(open('$MAP'))['targets']['$key'].get('$1',''))"; } + +sc2_gate="$(mapget sc2_gate)" +test_cmd="$(mapget test_cmd)" + +# --- gate target (config-audit): deterministic, strict by design --- +if [ -n "$sc2_gate" ]; then + if WORK="$WORK" bash "$SCRIPT_DIR/$sc2_gate" >/dev/null 2>&1; then + echo "$key: PASS ($sc2_gate)"; exit 0 + fi + echo "$key: FAIL ($sc2_gate)" >&2; exit 1 +fi + +# --- strict standalone validation first (SC2 + SC7 + per-target structural checks) --- +if WORK="$WORK" bash "$SCRIPT_DIR/40-validate-standalone.sh" "$key" >/dev/null 2>&1; then + echo "$key: PASS (standalone strict)"; exit 0 +fi + +# Strict failed. Only a node:test suite can fail regression-relative-acceptably; a structure validator +# failing is a genuine structural defect — do not soften it. +case "$test_cmd" in + *node\ --test*) : ;; + *) echo "$key: FAIL (standalone strict; structure validator — not regression-eligible)" >&2; exit 1 ;; +esac + +dest="$WORK/$key" +[ -d "$dest/.git" ] || { echo "$key: FAIL (no prepped extract at $dest — run 40-validate-standalone.sh first)" >&2; exit 1; } + +# Regression-relative: the standalone failing-NAME set must be a SUBSET of the live in-repo failing-NAME set. +# Capture the standalone set from the PREPPED EXTRACT ($WORK/$key), NOT 40's /tmp/claude-$key side-effect +# clean room (4e494c8: that coupling masked a real regression when the dir was absent). The subset decision +# is delegated to sc2-regression.sh; a missing capture FILE there is a hard error, never a false PASS. +sf="$(mktemp)" && bf="$(mktemp)" || { echo "$key: FAIL (mktemp)" >&2; exit 1; } +trap 'rm -f "$sf" "$bf"' EXIT +bash "$SCRIPT_DIR/capture-fails.sh" "$dest" "$test_cmd" > "$sf" 2>/dev/null +bash "$SCRIPT_DIR/capture-fails.sh" "$REPO_ROOT/plugins/$key" "$test_cmd" > "$bf" 2>/dev/null +pre="$(wc -l < "$bf" | tr -d '[:space:]')" + +if regr="$(bash "$SCRIPT_DIR/sc2-regression.sh" "$sf" "$bf")"; then + echo "$key: PASS (${pre} pre-existing, regression-relative)" + exit 0 +else + rc=$? + if [ "$rc" -ge 2 ]; then + echo "$key: FAIL (sc2-regression error — missing capture file)" >&2 + else + echo "$key: FAIL (regression: $(printf '%s' "$regr" | grep -c .) new failure(s) absent from in-repo baseline):" >&2 + printf '%s\n' "$regr" >&2 + fi + exit 1 +fi diff --git a/docs/marketplace-polyrepo-migration/migration/41-validate-or-regression.test.mjs b/docs/marketplace-polyrepo-migration/migration/41-validate-or-regression.test.mjs new file mode 100644 index 0000000..58c392c --- /dev/null +++ b/docs/marketplace-polyrepo-migration/migration/41-validate-or-regression.test.mjs @@ -0,0 +1,141 @@ +// Coverage for 41-validate-or-regression.sh — the per-target gate the operator window calls in step [a]. +// It enforces the migration's RATIFIED contract ("introduce no regression"), NOT the stricter zero-failure +// gate that made run-operator-window.sh STOP on voyage's / ai-psychosis's pre-existing in-repo red. 41 is +// thin glue over already-tested detectors (sc2-regression.sh — sc-checks.test.mjs; capture-fails.sh — +// capture-fails.test.mjs) and over 40-validate-standalone.sh (strict). This suite drives the REAL 41 inside +// a self-contained fake migration tree: the heavy extract pipeline (40) is STUBBED so the regression-branch +// glue is exercised hermetically, with no /tmp/polyrepo-migration state and no real plugin extracts. +// +// Branches covered: strict-PASS passthrough; regression-relative PASS (standalone ⊆ in-repo); genuine +// regression FAIL (standalone-only failure); structure-validator FAIL is NOT regression-eligible; the +// sc2_gate target stays strict (PASS + FAIL). +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync, mkdirSync, copyFileSync, chmodSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = path.dirname(fileURLToPath(import.meta.url)); + +// One node:test file = imports ONCE + a failing test() per name. (Concatenating per-name single-file +// strings would duplicate `import test` → SyntaxError → a file-level not-ok instead of per-test names.) +const failSuite = (names) => + "import test from 'node:test';\nimport assert from 'node:assert/strict';\n" + + names.map((n) => `test(${JSON.stringify(n)}, () => assert.equal(1, 2));`).join('\n') + + '\n'; + +function write(p, content, mode) { + mkdirSync(path.dirname(p), { recursive: true }); + writeFileSync(p, content); + if (mode) chmodSync(p, mode); +} + +// Build a fake migration tree: SCRIPT_DIR nested 3 deep so 41's REPO_ROOT ($SCRIPT_DIR/../../..) is the root. +function buildTree() { + const root = mkdtempSync(path.join(os.tmpdir(), 'mig41-')); + const mig = path.join(root, 'a', 'b', 'c'); // SCRIPT_DIR; ../../.. === root + const work = path.join(root, '_work'); // $WORK + mkdirSync(mig, { recursive: true }); + + // real scripts under test + reused detectors + for (const f of ['41-validate-or-regression.sh', 'capture-fails.sh', 'sc2-regression.sh']) { + copyFileSync(path.join(here, f), path.join(mig, f)); + } + + // stub strict 40-validate-standalone.sh: passes ONLY for the designated strict-pass target. + write(path.join(mig, '40-validate-standalone.sh'), + '#!/usr/bin/env bash\n[ "${1:-}" = "strictpass" ] && exit 0\nexit 1\n', 0o755); + // sc2_gate stubs + write(path.join(mig, 'gate-pass.sh'), '#!/usr/bin/env bash\nexit 0\n', 0o755); + write(path.join(mig, 'gate-fail.sh'), '#!/usr/bin/env bash\nexit 1\n', 0o755); + + const NT = "node --test 'tests/**/*.test.mjs'"; + const map = { + targets: { + strictpass: { test_cmd: NT }, + subset: { test_cmd: NT }, + regress: { test_cmd: NT }, + structure: { test_cmd: 'bash validate-plugin.sh' }, + gatepass: { test_cmd: NT, sc2_gate: 'gate-pass.sh' }, + gatefail: { test_cmd: NT, sc2_gate: 'gate-fail.sh' }, + }, + }; + write(path.join(mig, 'plugin-map.json'), JSON.stringify(map, null, 2)); + + // node:test suites for the regression-eligible targets. dest = $WORK/ (needs .git); baseline = root/plugins/. + const suite = (key, names) => { + mkdirSync(path.join(work, key, '.git'), { recursive: true }); + write(path.join(work, key, 'tests', 'x.test.mjs'), failSuite(names.standalone)); + write(path.join(root, 'plugins', key, 'tests', 'x.test.mjs'), failSuite(names.baseline)); + }; + suite('subset', { standalone: ['shared red'], baseline: ['shared red', 'other in-repo red'] }); + suite('regress', { standalone: ['shared red', 'extract regressed'], baseline: ['shared red'] }); + + return { root, mig, work }; +} + +function run41(tree, key) { + // Strip NODE_TEST_CONTEXT: 41 → capture-fails → `node --test`, which emits no TAP if it inherits the + // harness's nested-test context (Step-6 env-clean gotcha). The real window runs as plain bash, unnested. + const env = { ...process.env, WORK: tree.work }; + delete env.NODE_TEST_CONTEXT; + return spawnSync('bash', [path.join(tree.mig, '41-validate-or-regression.sh'), key], + { encoding: 'utf8', env }); +} + +test('strict-PASS passthrough: 40 passes → 41 PASS (standalone strict), exit 0, no capture', () => { + const tree = buildTree(); + try { + const r = run41(tree, 'strictpass'); + assert.equal(r.status, 0, `strict pass must exit 0: ${r.stderr}`); + assert.match(r.stdout, /PASS \(standalone strict\)/, r.stdout); + } finally { rmSync(tree.root, { recursive: true, force: true }); } +}); + +test('regression-relative PASS: standalone failing-set ⊆ in-repo → exit 0, reports N pre-existing', () => { + const tree = buildTree(); + try { + const r = run41(tree, 'subset'); + assert.equal(r.status, 0, `a subset of in-repo red is not a migration regression: ${r.stdout}${r.stderr}`); + assert.match(r.stdout, /pre-existing, regression-relative/, r.stdout); + } finally { rmSync(tree.root, { recursive: true, force: true }); } +}); + +test('genuine regression FAIL: a standalone-only failure (absent in-repo) → non-zero, names the regression', () => { + const tree = buildTree(); + try { + const r = run41(tree, 'regress'); + assert.notEqual(r.status, 0, 'a new failure introduced by extraction must STOP the window'); + assert.match(r.stderr, /regression/, r.stderr); + assert.match(r.stderr, /extract regressed/, `the regressing test name must be surfaced: ${r.stderr}`); + } finally { rmSync(tree.root, { recursive: true, force: true }); } +}); + +test('structure-validator FAIL is NOT regression-eligible: bash validator fail → non-zero, never softened', () => { + const tree = buildTree(); + try { + const r = run41(tree, 'structure'); + assert.notEqual(r.status, 0, 'a deterministic structure-validator failure is a genuine defect'); + assert.match(r.stderr, /not regression-eligible/, r.stderr); + } finally { rmSync(tree.root, { recursive: true, force: true }); } +}); + +test('sc2_gate target stays strict: gate PASS → exit 0', () => { + const tree = buildTree(); + try { + const r = run41(tree, 'gatepass'); + assert.equal(r.status, 0, r.stderr); + assert.match(r.stdout, /PASS \(gate-pass\.sh\)/, r.stdout); + } finally { rmSync(tree.root, { recursive: true, force: true }); } +}); + +test('sc2_gate target stays strict: gate FAIL → non-zero', () => { + const tree = buildTree(); + try { + const r = run41(tree, 'gatefail'); + assert.notEqual(r.status, 0, 'a failing dedicated gate must STOP the window'); + assert.match(r.stderr, /FAIL \(gate-fail\.sh\)/, r.stderr); + } finally { rmSync(tree.root, { recursive: true, force: true }); } +}); diff --git a/docs/marketplace-polyrepo-migration/migration/99-dryrun.sh b/docs/marketplace-polyrepo-migration/migration/99-dryrun.sh index 30ba49b..f85cedf 100644 --- a/docs/marketplace-polyrepo-migration/migration/99-dryrun.sh +++ b/docs/marketplace-polyrepo-migration/migration/99-dryrun.sh @@ -59,14 +59,9 @@ live_files() { echo "$total" } -# Failing-test NAME set for a node:test suite, via the stable TAP reporter (locale-independent). -# $1 = dir to run in, $2 = the plugin's node --test command. -capture_fails() { - local dir="$1" cmd="$2" tapcmd - tapcmd="${cmd/node --test/node --test --test-reporter=tap}" - ( cd "$dir" && eval "$tapcmd" ) 2>&1 \ - | grep -E '^not ok ' | sed -E 's/^not ok [0-9]+ - //; s/ #.*$//' | sort -u -} +# Failing-test NAME set for a node:test suite — delegated to capture-fails.sh so the SAME capture feeds both +# this rehearsal and the live operator-window gate (41-validate-or-regression.sh). $1 = dir, $2 = node --test cmd. +capture_fails() { bash "$SCRIPT_DIR/capture-fails.sh" "$1" "$2"; } [ -f "$MAP" ] || fail "plugin-map.json missing at $MAP" mkdir -p "$WORK" diff --git a/docs/marketplace-polyrepo-migration/migration/RUNBOOK.md b/docs/marketplace-polyrepo-migration/migration/RUNBOOK.md index b4346c5..e01f6c0 100644 --- a/docs/marketplace-polyrepo-migration/migration/RUNBOOK.md +++ b/docs/marketplace-polyrepo-migration/migration/RUNBOOK.md @@ -99,11 +99,15 @@ Order (lower risk last so problems surface while the blast radius is small): ```bash KEY= # e.g. voyage -# a. extract + validate standalone (config-audit uses its dedicated SC2 gate) +# a. extract + validate standalone (config-audit uses its dedicated SC2 gate). +# SC2 is REGRESSION-RELATIVE (the contract the Step-11 dry-run validated): 41 runs 40 strict, then on +# failure passes iff the standalone failing-test set is a SUBSET of the live in-repo set — so pre-existing +# in-repo red (e.g. voyage's 2 doc-consistency drifts, ai-psychosis's 1) does NOT STOP the rollout, while +# a genuine extraction-introduced regression still does. (Strict 40 alone would STOP on that blessed red.) if [ "$KEY" = "config-audit" ]; then WORK=$WORK bash MIG/50-config-audit-sc2.sh # config-audit: SC2 PASS (...) else - WORK=$WORK bash MIG/40-validate-standalone.sh "$KEY" # : PASS (...) + WORK=$WORK bash MIG/41-validate-or-regression.sh "$KEY" # : PASS (standalone strict | N pre-existing) fi # b. create the Forgejo repo (auto_init:false, public) diff --git a/docs/marketplace-polyrepo-migration/migration/capture-fails.sh b/docs/marketplace-polyrepo-migration/migration/capture-fails.sh new file mode 100755 index 0000000..93ebf78 --- /dev/null +++ b/docs/marketplace-polyrepo-migration/migration/capture-fails.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Failing-test NAME set for a node:test suite, via the stable TAP reporter (locale-independent). +# Extracted from 99-dryrun.sh's inline capture_fails (mirrors the sc2-regression.sh extraction, 5d112cb) so +# the SAME capture feeds BOTH the dry-run rehearsal (99-dryrun.sh) and the live operator-window gate +# (41-validate-or-regression.sh) — a single source of truth prevents the gate and the rehearsal from +# disagreeing on what "failing" means (the exact class of drift that let the strict window STOP on red the +# dry-run had blessed). +# +# Prints sorted-unique failing test names, one per line. Empty output = the suite reported no `not ok` lines. +# The caller distinguishes "ran clean" from "did not run": sc2-regression.sh treats a missing capture FILE +# as a hard error, but an empty-yet-present file is a legitimate zero-failure set. No pipefail: a no-match +# grep (zero failures) must exit 0, not leak a spurious non-zero up to the caller. +# +# Usage: capture-fails.sh +set -u +dir="${1:?dir}" +cmd="${2:?node --test command}" +tapcmd="${cmd/node --test/node --test --test-reporter=tap}" +( cd "$dir" && eval "$tapcmd" ) 2>&1 \ + | grep -E '^not ok ' | sed -E 's/^not ok [0-9]+ - //; s/ #.*$//' | sort -u diff --git a/docs/marketplace-polyrepo-migration/migration/capture-fails.test.mjs b/docs/marketplace-polyrepo-migration/migration/capture-fails.test.mjs new file mode 100644 index 0000000..0333f8e --- /dev/null +++ b/docs/marketplace-polyrepo-migration/migration/capture-fails.test.mjs @@ -0,0 +1,73 @@ +// Coverage for capture-fails.sh — the failing-test-NAME capture extracted from 99-dryrun.sh so a SINGLE +// source feeds both the dry-run rehearsal and the live operator-window gate (41-validate-or-regression.sh). +// Proves it reports exactly the failing names (sorted, deduped across files) and stays exit-0 on an all-pass +// suite — a no-match grep must NOT leak a non-zero status up to the caller (which would masquerade as a +// failed capture). Drives the real script against tiny node:test fixtures — hermetic, no migration state. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const CAP = path.join(here, 'capture-fails.sh'); +const CMD = "node --test 'tests/**/*.test.mjs'"; + +// One node:test file = imports ONCE + a test() per entry. entry: { name, pass }. (Concatenating +// per-name single-file strings would duplicate `import test` → SyntaxError → a file-level not-ok, masking +// the per-test capture under test.) +const suiteFile = (entries) => + "import test from 'node:test';\nimport assert from 'node:assert/strict';\n" + + entries.map((e) => `test(${JSON.stringify(e.name)}, () => assert.equal(1, ${e.pass ? 1 : 2}));`).join('\n') + + '\n'; + +function sandbox(files) { + const dir = mkdtempSync(path.join(os.tmpdir(), 'capfail-')); + for (const [rel, content] of Object.entries(files)) { + const p = path.join(dir, rel); + mkdirSync(path.dirname(p), { recursive: true }); + writeFileSync(p, content); + } + return dir; +} + +function cap(dir) { + // Strip NODE_TEST_CONTEXT: capture-fails spawns `node --test`, which misbehaves (emits no TAP) when it + // inherits the harness's nested-test context (Step-6 env-clean gotcha). The real operator window runs as + // plain bash, never nested under node:test, so this only matters inside this harness. + const env = { ...process.env }; + delete env.NODE_TEST_CONTEXT; + return spawnSync('bash', [CAP, dir, CMD], { encoding: 'utf8', env }); +} + +test('capture-fails reports the failing test name (and only it), exit 0', () => { + const dir = sandbox({ 'tests/a.test.mjs': suiteFile([{ name: 'good', pass: true }, { name: 'bad apple', pass: false }]) }); + try { + const r = cap(dir); + assert.equal(r.status, 0, `capture must exit 0 even with failures: ${r.stderr}`); + assert.equal(r.stdout.trim(), 'bad apple', `only the failing name expected, got: ${JSON.stringify(r.stdout)}`); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +test('capture-fails on an all-pass suite → empty output, exit 0 (no-match grep must not leak non-zero)', () => { + const dir = sandbox({ 'tests/a.test.mjs': suiteFile([{ name: 'good', pass: true }, { name: 'also good', pass: true }]) }); + try { + const r = cap(dir); + assert.equal(r.status, 0, `all-pass must exit 0: ${r.stderr}`); + assert.equal(r.stdout.trim(), '', `no failing names expected, got: ${JSON.stringify(r.stdout)}`); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +test('capture-fails sorts + dedups failing names across files', () => { + const dir = sandbox({ + 'tests/a.test.mjs': suiteFile([{ name: 'zeta', pass: false }, { name: 'alpha', pass: false }]), + 'tests/b.test.mjs': suiteFile([{ name: 'alpha', pass: false }]), // duplicate name across files collapses to one + }); + try { + const r = cap(dir); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout.trim(), 'alpha\nzeta', `sorted+deduped expected, got: ${JSON.stringify(r.stdout)}`); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/docs/marketplace-polyrepo-migration/migration/run-operator-window.sh b/docs/marketplace-polyrepo-migration/migration/run-operator-window.sh index 3936b2d..7755360 100644 --- a/docs/marketplace-polyrepo-migration/migration/run-operator-window.sh +++ b/docs/marketplace-polyrepo-migration/migration/run-operator-window.sh @@ -81,12 +81,19 @@ rollout_one() { say "" say "==== $key (tag $tag) ====" - # (a) validate the extract — self-extracts if $WORK/$key is absent + # (a) validate the extract — self-extracts if $WORK/$key is absent. + # SC2 is REGRESSION-RELATIVE (the contract the Step-11 dry-run validated): a target passes iff the + # extraction introduces NO NEW failure. Pre-existing in-repo red (voyage's 2 doc-consistency drifts, + # ai-psychosis's 1) is the plugin's own concern — 41-validate-or-regression.sh enforces that exact + # contract (strict 40 first, then standalone-failing ⊆ in-repo-failing) so the window does NOT STOP on + # red the rehearsal blessed. config-audit keeps its dedicated deterministic gate. say " [a] validate extract…" if [ "$key" = "config-audit" ]; then WORK="$WORK" bash "$MIG/50-config-audit-sc2.sh" >/dev/null 2>&1 || die "$key SC2 gate FAILED — STOP" else - WORK="$WORK" bash "$MIG/40-validate-standalone.sh" "$key" >/dev/null 2>&1 || die "$key standalone validation FAILED — STOP" + sc2out="$(WORK="$WORK" bash "$MIG/41-validate-or-regression.sh" "$key" 2>&1)" \ + || die "$key standalone validation FAILED (incl. regression-relative SC2) — STOP${sc2out:+ :: $sc2out}" + case "$sc2out" in *pre-existing*) say " ${sc2out#*: }";; esac fi # (b) create the Forgejo repo (201 created | 409 already exists)