chore(migration): full local dry-run harness + report
This commit is contained in:
parent
836b8e9337
commit
5e00f92786
2 changed files with 327 additions and 0 deletions
238
docs/marketplace-polyrepo-migration/migration/99-dryrun.sh
Normal file
238
docs/marketplace-polyrepo-migration/migration/99-dryrun.sh
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
#!/usr/bin/env bash
|
||||
# Step 11 — Full local dry-run (all 11 targets, NULL push).
|
||||
#
|
||||
# The single end-to-end rehearsal of the Claude-run local half. For all 11 targets (10 plugins + the
|
||||
# design-system) it FORCE-FRESH re-extracts (Steps 3->4->5: never reuses a possibly-stale $WORK/<key>),
|
||||
# then validates each three ways:
|
||||
# SC6 (content retention) — extract git-tracked file count vs the live monorepo's tracked count for the
|
||||
# target's paths. A drop (not explained by an intentional blob-strip) is a hard FAIL: this is the
|
||||
# check that catches the llm-security dual-rename class of defect (87 files dropped) deterministically.
|
||||
# SC7 (no state-file leak) — no tracked STATE.md / *.local.md (except the okr template).
|
||||
# SC2 (no test REGRESSION) — config-audit routes to its dedicated Step-7 gate (50-config-audit-sc2.sh);
|
||||
# every other target runs its declared standalone runner via 40-validate-standalone.sh. If the
|
||||
# standalone runner fails, the gate is REGRESSION-RELATIVE: it re-runs the same suite in the live
|
||||
# monorepo and PASSES iff the standalone failing-test set is a SUBSET of the in-repo failing set
|
||||
# (i.e. the extraction introduced NO new failures). Pre-existing in-repo red and environmental
|
||||
# flake are the plugin's own concern, NOT a migration regression, and are reported transparently
|
||||
# (never masked) — the migration's contract is "introduce no regression".
|
||||
# Then it exercises Step 8's marketplace rewriter (--all -> --out) and Step 9's catalog thinning
|
||||
# (--workspace), and runs the cross-repo DS-vendor chain (M9 / SC5): sync playground-design-system
|
||||
# (--source) into ms-ai-architect (--target), then --check for byte-parity. It writes
|
||||
# $WORK/dryrun-report.md (an OUTPUT artifact, never committed).
|
||||
#
|
||||
# Reuses the dedicated `git clone --no-local` extraction mirror ($WORK/_mirror, R4/M6) so extraction never
|
||||
# reads the working checkout. NULL push (D8): zero pushes, zero live-file mutation — the live plugins/,
|
||||
# shared/, and .claude-plugin/marketplace.json are never touched; every write lands in $WORK. Honest scope
|
||||
# (M8): this proves the LOCAL mechanics only; the window-only steps (Forgejo accepting post-strip history,
|
||||
# auto_init:false API repo creation, Claude Code resolving an HTTPS url+ref source) are gated by the
|
||||
# Step 10 pilot, NOT by this dry-run.
|
||||
#
|
||||
# On any hard FAIL (SC6 drop, SC7 leak, or an SC2 regression): exit non-zero (escalate — never mask a
|
||||
# real pre-window blocker).
|
||||
#
|
||||
# Usage: 99-dryrun.sh (override WORK= to relocate the workspace; default /tmp/polyrepo-migration)
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
MAP="$SCRIPT_DIR/plugin-map.json"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
WORK="${WORK:-/tmp/polyrepo-migration}"
|
||||
MIRROR="$WORK/_mirror"
|
||||
REPORT="$WORK/dryrun-report.md"
|
||||
MP_PREVIEW="$WORK/marketplace.all-external.json"
|
||||
THIN_WS="$WORK/_thin-catalog"
|
||||
SYNC="$REPO_ROOT/scripts/sync-design-system.mjs"
|
||||
|
||||
fail() { printf 'DRY-RUN FAIL: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
mapget() { python3 -c "import json;print(json.load(open('$MAP'))['targets']['$1'].get('$2',''))"; }
|
||||
mappaths() { python3 -c "import json;print(' '.join(json.load(open('$MAP'))['targets']['$1']['paths']))"; }
|
||||
|
||||
# Live monorepo git-tracked file count across a target's source paths (SC6 baseline).
|
||||
live_files() {
|
||||
local key="$1" total=0 p n
|
||||
for p in $(mappaths "$key"); do
|
||||
n="$(git -C "$REPO_ROOT" ls-files "$p" | wc -l | tr -d '[:space:]')"
|
||||
total=$((total + n))
|
||||
done
|
||||
echo "$total"
|
||||
}
|
||||
|
||||
# Failing-test NAME set for a node:test suite, via the stable TAP reporter (locale-independent).
|
||||
# $1 = dir to run in, $2 = the plugin's node --test command.
|
||||
capture_fails() {
|
||||
local dir="$1" cmd="$2" tapcmd
|
||||
tapcmd="${cmd/node --test/node --test --test-reporter=tap}"
|
||||
( cd "$dir" && eval "$tapcmd" ) 2>&1 \
|
||||
| grep -E '^not ok ' | sed -E 's/^not ok [0-9]+ - //; s/ #.*$//' | sort -u
|
||||
}
|
||||
|
||||
[ -f "$MAP" ] || fail "plugin-map.json missing at $MAP"
|
||||
mkdir -p "$WORK"
|
||||
|
||||
KEYS="$(python3 -c "import json;print('\n'.join(sorted(json.load(open('$MAP'))['targets'])))")" \
|
||||
|| fail "plugin-map.json does not parse"
|
||||
TARGET_COUNT="$(printf '%s\n' "$KEYS" | grep -c .)"
|
||||
[ "$TARGET_COUNT" = "11" ] || fail "expected 11 targets, found $TARGET_COUNT"
|
||||
|
||||
# Ensure the dedicated --no-local mirror exists (R4/M6); extraction never reads the working checkout.
|
||||
if [ ! -e "$MIRROR/HEAD" ] && [ ! -d "$MIRROR/.git" ]; then
|
||||
echo " mirror absent -> running 00-preflight.sh (builds the --no-local extraction mirror)"
|
||||
WORK="$WORK" bash "$SCRIPT_DIR/00-preflight.sh" >/dev/null || fail "preflight (mirror build) failed"
|
||||
fi
|
||||
|
||||
{
|
||||
echo "# Polyrepo Migration — Local Dry-Run Report"
|
||||
echo
|
||||
echo "> Output artifact (NOT committed). Workspace: \`$WORK\`. Force-fresh extraction (no stale reuse)."
|
||||
echo "> **NULL push (D8):** 0 pushes, 0 live-file mutation (plugins/, shared/, marketplace.json untouched)."
|
||||
echo "> Extraction reuses the dedicated \`git clone --no-local\` mirror (R4/M6) — never the working checkout."
|
||||
echo "> **SC2 is regression-relative:** a target passes iff the extraction introduces NO test failure that"
|
||||
echo "> does not also occur in the live monorepo. Pre-existing in-repo red / flake is reported, never masked."
|
||||
echo
|
||||
echo "## Per-target — force-fresh extraction + standalone validation (Steps 3→4→5, +6/+7)"
|
||||
echo
|
||||
echo "| target | commits | files ext/live | tag | SC2 | SC7 |"
|
||||
echo "|--------|--------:|----------------|-----|-----|-----|"
|
||||
} > "$REPORT"
|
||||
|
||||
PASSED=0
|
||||
for key in $KEYS; do
|
||||
dest="$WORK/$key"
|
||||
ok=1
|
||||
|
||||
# --- force-fresh extract + prep (Steps 3->4->5; 10-extract wipes $dest first) ---
|
||||
WORK="$WORK" bash "$SCRIPT_DIR/10-extract.sh" "$key" >/dev/null 2>&1 || ok=0
|
||||
WORK="$WORK" bash "$SCRIPT_DIR/20-rehome-config.sh" "$key" >/dev/null 2>&1 || ok=0
|
||||
WORK="$WORK" node "$SCRIPT_DIR/30-fix-references.mjs" "$key" >/dev/null 2>&1 || ok=0
|
||||
|
||||
# --- report data + SC6 + SC7 from the fresh extract ---
|
||||
commits=0; tag="(none)"; ext_files=0; sc7="clean"; sc6="?"
|
||||
if [ -d "$dest/.git" ]; then
|
||||
commits="$(git -C "$dest" rev-list --count HEAD 2>/dev/null || echo 0)"
|
||||
tag="$(git -C "$dest" tag 2>/dev/null | head -1)"; tag="${tag:-(none)}"
|
||||
ext_files="$(git -C "$dest" ls-files | wc -l | tr -d '[:space:]')"
|
||||
leak="$(git -C "$dest" ls-files | grep -E 'STATE\.md|\.local\.md$' \
|
||||
| grep -v 'templates/okr\.local\.md\.template' || true)"
|
||||
[ -z "$leak" ] || { sc7="LEAK: $(printf '%s' "$leak" | tr '\n' ' ')"; ok=0; }
|
||||
else
|
||||
ok=0
|
||||
fi
|
||||
|
||||
# SC6 content retention. Intentional blob-strip targets may legitimately shed >1MB blobs, so a drop
|
||||
# there is reported, not failed; for every other target a drop is a hard FAIL (the llm-security class).
|
||||
lf="$(live_files "$key")"
|
||||
blob="$(mapget "$key" blob_strip)"
|
||||
if [ "$ext_files" -lt "$lf" ]; then
|
||||
if [ "$blob" = "True" ]; then sc6="${ext_files}/${lf} (blob-strip)"; else sc6="${ext_files}/${lf} DROP"; ok=0; fi
|
||||
else
|
||||
sc6="${ext_files}/${lf}"
|
||||
fi
|
||||
|
||||
# --- SC2 (regression-relative) ---
|
||||
sc2_gate="$(mapget "$key" sc2_gate)"
|
||||
test_cmd="$(mapget "$key" test_cmd)"
|
||||
if [ -n "$sc2_gate" ]; then
|
||||
WORK="$WORK" bash "$SCRIPT_DIR/$sc2_gate" >/dev/null 2>&1 && sc2="PASS" || { sc2="FAIL"; ok=0; }
|
||||
else
|
||||
if WORK="$WORK" bash "$SCRIPT_DIR/40-validate-standalone.sh" "$key" >/dev/null 2>&1; then
|
||||
sc2="PASS"
|
||||
else
|
||||
case "$test_cmd" in
|
||||
*node\ --test*)
|
||||
# Regression-relative: standalone failing set must be a subset of the in-repo failing set.
|
||||
sf="$(mktemp)"; bf="$(mktemp)"
|
||||
capture_fails "/tmp/claude-$key" "$test_cmd" > "$sf" 2>/dev/null
|
||||
capture_fails "$REPO_ROOT/plugins/$key" "$test_cmd" > "$bf" 2>/dev/null
|
||||
regr="$(comm -23 "$sf" "$bf")"
|
||||
pre="$(wc -l < "$bf" | tr -d '[:space:]')"
|
||||
rm -f "$sf" "$bf"
|
||||
if [ -z "$regr" ]; then sc2="PASS (${pre} pre-existing)"; else
|
||||
sc2="FAIL (regression: $(printf '%s' "$regr" | grep -c .))"; ok=0; fi
|
||||
;;
|
||||
*) sc2="FAIL (structure)"; ok=0 ;; # deterministic structure check genuinely failed
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
|
||||
printf '| %s | %s | %s | %s | %s | %s |\n' "$key" "$commits" "$sc6" "$tag" "$sc2" "$sc7" >> "$REPORT"
|
||||
echo " $key: commits=$commits files=$sc6 sc2=$sc2 sc7=${sc7%%:*}"
|
||||
[ "$ok" -eq 1 ] && PASSED=$((PASSED+1))
|
||||
done
|
||||
|
||||
# --- Step 8: all-external marketplace.json preview (F2 — HTTPS + ref, zero local) ---
|
||||
if node "$SCRIPT_DIR/60-rewrite-marketplace.mjs" --all --out "$MP_PREVIEW" >/dev/null 2>&1; then
|
||||
LOCAL_REFS="$(grep -c '\./plugins/' "$MP_PREVIEW" 2>/dev/null || true)"; LOCAL_REFS="${LOCAL_REFS:-0}"
|
||||
SSH_REFS="$(grep -c 'ssh://' "$MP_PREVIEW" 2>/dev/null || true)"; SSH_REFS="${SSH_REFS:-0}"
|
||||
EXT_REFS="$(grep -c '"source": "url"' "$MP_PREVIEW" 2>/dev/null || true)"; EXT_REFS="${EXT_REFS:-0}"
|
||||
else
|
||||
LOCAL_REFS="?"; SSH_REFS="?"; EXT_REFS="?"
|
||||
fi
|
||||
|
||||
# --- Step 9: thin-catalog preview (SC4 — plugins/ & shared/ removed, CONVENTIONS.md extracted) ---
|
||||
THIN_OK="yes"
|
||||
if bash "$SCRIPT_DIR/70-thin-catalog.sh" --workspace "$THIN_WS" >/dev/null 2>&1; then
|
||||
[ -d "$THIN_WS/plugins" ] && THIN_OK="no (plugins/ remain)"
|
||||
[ -d "$THIN_WS/shared" ] && THIN_OK="no (shared/ remain)"
|
||||
[ -f "$THIN_WS/CONVENTIONS.md" ] || THIN_OK="no (CONVENTIONS.md missing)"
|
||||
else
|
||||
THIN_OK="no (script error)"
|
||||
fi
|
||||
|
||||
# --- M9 / SC5: cross-repo DS-vendor chain (the gap the pilot can't cover) ---
|
||||
DS="$WORK/playground-design-system"; MSA="$WORK/ms-ai-architect"
|
||||
DS_VENDOR="skipped (DS or ms-ai-architect extract missing)"
|
||||
if [ -d "$DS/.git" ] && [ -d "$MSA/.git" ] && [ -f "$SYNC" ]; then
|
||||
if node "$SYNC" ms-ai-architect --source "$DS" --target "$MSA" >/dev/null 2>&1 \
|
||||
|| node "$SYNC" ms-ai-architect --source "$DS" --target "$MSA" --force >/dev/null 2>&1; then
|
||||
CHK="$(node "$SYNC" ms-ai-architect --source "$DS" --target "$MSA" --check 2>&1)"
|
||||
if [ $? -eq 0 ]; then DS_VENDOR="OK — $CHK"; else DS_VENDOR="DRIFT — $CHK"; fi
|
||||
else
|
||||
DS_VENDOR="sync failed"
|
||||
fi
|
||||
fi
|
||||
|
||||
ALL_OK=1
|
||||
[ "$PASSED" = "11" ] || ALL_OK=0
|
||||
[ "$LOCAL_REFS" = "0" ] || ALL_OK=0
|
||||
[ "$SSH_REFS" = "0" ] || ALL_OK=0
|
||||
[ "$THIN_OK" = "yes" ] || ALL_OK=0
|
||||
case "$DS_VENDOR" in OK*) : ;; *) ALL_OK=0 ;; esac
|
||||
|
||||
{
|
||||
echo
|
||||
echo "## marketplace.json preview — all external (Step 8, F2)"
|
||||
echo
|
||||
echo "- out: \`$MP_PREVIEW\` · local \`./plugins/\`: $LOCAL_REFS (want 0) · \`ssh://\`: $SSH_REFS (want 0) · external (https+ref): $EXT_REFS"
|
||||
echo
|
||||
echo "## thin-catalog preview (Step 9, SC4)"
|
||||
echo
|
||||
echo "- workspace: \`$THIN_WS\` · plugins/ & shared/ removed + CONVENTIONS.md extracted: $THIN_OK"
|
||||
echo
|
||||
echo "## DS-vendor cross-repo validation (M9 / SC5)"
|
||||
echo
|
||||
echo "- \`playground-design-system\` (--source) → \`ms-ai-architect\` (--target) → \`--check\`: $DS_VENDOR"
|
||||
echo
|
||||
echo "## Honest scope of this dry-run (M8)"
|
||||
echo
|
||||
echo "Proves the **local mechanics**: force-fresh rename-aware extraction + content retention (SC6),"
|
||||
echo "re-rooting, reference rewrites, regression-relative standalone validation (SC2/SC7), the"
|
||||
echo "marketplace/catalog transforms, and the DS-vendor chain. It does **not** prove the **window-only**"
|
||||
echo "steps (Forgejo push of post-strip history, \`auto_init:false\` API repo creation, Claude Code"
|
||||
echo "resolving an HTTPS \`url\`+\`ref\` source). Those are gated by the **Step 10 pilot**, not this dry-run."
|
||||
echo
|
||||
if [ "$ALL_OK" = "1" ]; then
|
||||
echo "**DRY-RUN OK — $PASSED/11 targets extracted + validated, 0 pushes**"
|
||||
else
|
||||
echo "**DRY-RUN INCOMPLETE — $PASSED/11 targets passed; inspect the rows above**"
|
||||
fi
|
||||
} >> "$REPORT"
|
||||
|
||||
echo " report: $REPORT"
|
||||
if [ "$ALL_OK" = "1" ]; then
|
||||
echo "DRY-RUN OK — 11/11 targets extracted + validated, 0 pushes"
|
||||
exit 0
|
||||
else
|
||||
echo "DRY-RUN INCOMPLETE — $PASSED/11 targets passed; see $REPORT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
// Step 11 test — the full local dry-run report ($WORK/dryrun-report.md) + previews.
|
||||
// Verifies the GENERATED artifact (does not re-derive it):
|
||||
// - the report lists all 11 targets
|
||||
// - voyage shows >=250 commits (F1 — pre-rename history retained)
|
||||
// - config-audit shows PASS under its dedicated SC2 gate (50-config-audit-sc2.sh)
|
||||
// - llm-security shows complete content retention (SC6 — no file DROP; the dual-rename defect is fixed)
|
||||
// - no SC6 DROP and no SC2 regression on ANY target
|
||||
// - no target shows a tracked state-file leak (SC7)
|
||||
// - the previewed all-external marketplace.json has zero ./plugins/ and zero ssh:// (F2)
|
||||
// Pattern: plugins/voyage/tests/integration/*.test.mjs (heavy integration — long timeout, artifact reuse).
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const SCRIPT = path.join(here, '99-dryrun.sh');
|
||||
const WORK = process.env.WORK || '/tmp/polyrepo-migration';
|
||||
const REPORT = path.join(WORK, 'dryrun-report.md');
|
||||
const MP_PREVIEW = path.join(WORK, 'marketplace.all-external.json');
|
||||
const LONG = 1000 * 60 * 25; // cold path force-fresh re-extracts 11 repos + runs their suites
|
||||
|
||||
const TARGETS = [
|
||||
'ai-psychosis', 'claude-design', 'config-audit', 'graceful-handoff',
|
||||
'human-friendly-style', 'linkedin-studio', 'llm-security', 'ms-ai-architect',
|
||||
'okr', 'playground-design-system', 'voyage',
|
||||
];
|
||||
|
||||
let _report = null;
|
||||
function report() {
|
||||
if (_report !== null) return _report;
|
||||
if (!existsSync(REPORT)) {
|
||||
const r = spawnSync('bash', [SCRIPT], { encoding: 'utf8', timeout: 1000 * 60 * 24, env: { ...process.env, WORK } });
|
||||
if (!existsSync(REPORT)) {
|
||||
throw new Error(`dry-run did not produce ${REPORT}\nstdout:\n${r.stdout}\nstderr:\n${r.stderr}`);
|
||||
}
|
||||
}
|
||||
_report = readFileSync(REPORT, 'utf8');
|
||||
return _report;
|
||||
}
|
||||
function row(key) {
|
||||
return report().split('\n').find((l) => new RegExp(`^\\|\\s*${key}\\s*\\|`).test(l)) || '';
|
||||
}
|
||||
|
||||
test('report lists all 11 targets', { timeout: LONG }, () => {
|
||||
const r = report();
|
||||
for (const t of TARGETS) assert.ok(r.includes(t), `report must list target ${t}`);
|
||||
const rows = r.split('\n').filter((l) => /^\|\s*[a-z0-9-]+\s*\|\s*\d+\s*\|/.test(l));
|
||||
assert.equal(rows.length, 11, `expected 11 target rows, found ${rows.length}`);
|
||||
});
|
||||
|
||||
test('voyage shows >=250 commits (F1 pre-rename history retained)', { timeout: LONG }, () => {
|
||||
const m = report().match(/\|\s*voyage\s*\|\s*(\d+)\s*\|/);
|
||||
assert.ok(m, 'voyage row with a commit count must be present');
|
||||
assert.ok(Number(m[1]) >= 250, `voyage commits ${m && m[1]} must be >= 250`);
|
||||
});
|
||||
|
||||
test('config-audit PASSes its dedicated SC2 gate', { timeout: LONG }, () => {
|
||||
const line = row('config-audit');
|
||||
assert.ok(line, 'config-audit row must be present');
|
||||
assert.ok(/\bPASS\b/.test(line) && !/FAIL/.test(line), `config-audit must PASS: ${line}`);
|
||||
});
|
||||
|
||||
test('llm-security retains all content (SC6 — dual-rename file-drop defect is fixed)', { timeout: LONG }, () => {
|
||||
const line = row('llm-security');
|
||||
assert.ok(line, 'llm-security row must be present');
|
||||
assert.ok(!/DROP/.test(line), `llm-security must not drop files (SC6): ${line}`);
|
||||
assert.ok(/\bPASS\b/.test(line) && !/FAIL/.test(line), `llm-security must PASS standalone: ${line}`);
|
||||
});
|
||||
|
||||
test('no SC6 file-drop and no SC2 regression on any target', { timeout: LONG }, () => {
|
||||
const r = report();
|
||||
assert.ok(!/\bDROP\b/.test(r), 'no target may drop content (SC6)');
|
||||
assert.ok(!/regression:/.test(r), 'no target may show an SC2 regression');
|
||||
});
|
||||
|
||||
test('no target shows a tracked state-file leak (SC7)', { timeout: LONG }, () => {
|
||||
assert.ok(!/LEAK/.test(report()), 'no SC7 state-file leak may appear in the report');
|
||||
});
|
||||
|
||||
test('all-external marketplace.json preview has zero ./plugins/ and zero ssh:// (F2)', { timeout: LONG }, () => {
|
||||
report();
|
||||
assert.ok(existsSync(MP_PREVIEW), `${MP_PREVIEW} must exist`);
|
||||
const mp = readFileSync(MP_PREVIEW, 'utf8');
|
||||
assert.ok(!mp.includes('./plugins/'), 'no local ./plugins/ source may remain');
|
||||
assert.ok(!mp.includes('ssh://'), 'no ssh:// url may remain');
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue