open/ktg-plugin-marketplace
1
0
Fork 0
Code Issues Releases Activity

fix(catalog): release gate can no longer be green without measuring anything

Browse source 
check-versions.mjs reported "12 SKIP, exit 0" on a fresh clone with no
sibling plugin repos — a run that verified nothing was indistinguishable
from a clean run. Any SKIP now fails the gate by default (--allow-skip
opts in explicitly), and the summary always reports "verified N/M".

Also adds a homepage-resolves check on plugin.json (dead-link ERROR,
verified live against voyage's stale pre-polyrepo URL); the field stays
optional since 11 of 12 plugins don't carry it yet.

gateOutcome is split out as a pure function so both directions (all-SKIP
fails, fully-verified-clean still passes) are unit-tested without needing
a real sibling-repo layout on disk.
This commit is contained in:
Kjell Tore Guttormsen 2026-08-26 11:22:09 +02:00
commit 0b0bf24137
3 changed files with 210 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

92
scripts/check-versions.mjs
View file

@ -7,10 +7,17 @@
// - plugin.json version must equal the README version-badge → else ERROR (internal)
// - catalog README per-plugin label must equal catalog ref → else ERROR (doc misstates install)
// - catalog ref should equal plugin.json version → else WARN (catalog lags / unreleased bump)
// - plugin.json homepage, if present, must resolve → else ERROR (dead link); field is OPTIONAL
// - sibling repo missing → SKIP
//
// Exit 1 if any ERROR (or any WARN under --strict); 0 otherwise.
// Usage: node scripts/check-versions.mjs [--strict]
// SKIP is UNVERIFIED, not clean — a plugin that could not be checked at all is not the
// same thing as a plugin that was checked and found fine. A run with any SKIP therefore
// exits 1 by default (fixes: 2026-08-18 a fresh clone with zero sibling repos printed
// "12 SKIP, exit 0" — a run that verified NOTHING read as green). Pass --allow-skip to
// accept partial verification explicitly; the summary line always reports "verified N/M".
//
// Exit 1 if any ERROR, any SKIP (unless --allow-skip), or any WARN under --strict; 0 otherwise.
// Usage: node scripts/check-versions.mjs [--strict] [--allow-skip]
import { readFileSync, existsSync } from 'node:fs';
import { execFileSync } from 'node:child_process';
import { join, dirname } from 'node:path';
@ -136,7 +143,7 @@ export function extractCatalogStats(readmeText, name) {
// Pure classifier — all I/O is resolved into the input shape before this is called.
// tags === null means "repo not inspected" (missing locally); [] means "no tags".
export function classifyPlugin({ name, catalogRef, pluginVersion, readmeBadge, tags, catalogLabel = null, statBadges = null, catalogStats = null }) {
export function classifyPlugin({ name, catalogRef, pluginVersion, readmeBadge, tags, catalogLabel = null, statBadges = null, catalogStats = null, homepage = null, homepageReachable = null }) {
if (pluginVersion === null && tags === null) {
return { name, status: 'SKIP', findings: [{ level: 'SKIP', msg: 'plugin repo not found locally — cannot verify' }] };
}
@ -192,6 +199,18 @@ export function classifyPlugin({ name, catalogRef, pluginVersion, readmeBadge, t
}
}
// 6. plugin.json homepage must resolve when present — the first link an agent follows
// from a manifest. The field stays OPTIONAL (measured 2026-08-18: 11 of 12 plugins omit
// it) — a deliberate policy call, not an oversight. Making it required would flip those
// 11 to ERROR and, via release-plugin.mjs's ERROR-set pre-flight (catalog-wide), block
// every release in the marketplace until each of those 11 sibling repos re-tags. That is
// a marketplace-wide freeze delivered by a script fix, not a gate improvement.
// `homepageReachable === null` means "not checked" (e.g. network unreachable) and must
// NOT read as either pass or fail.
if (homepage !== null && homepageReachable === false) {
findings.push({ level: 'ERROR', msg: `plugin.json homepage ${homepage} does not resolve (dead link)` });
}
const status = findings.some(f => f.level === 'ERROR') ? 'ERROR'
: findings.some(f => f.level === 'WARN') ? 'WARN'
: 'OK';
@ -219,6 +238,30 @@ function gitTags(repoDir) {
}
}
// Synchronous on purpose (curl via execFileSync, not fetch): inspectPlugin/runGate must
// stay synchronous because release-plugin.mjs's applyRelease calls `io.runGate(catalogDir)`
// without awaiting it. Turning runGate async would make that call site silently receive a
// Promise — preflightErrors reads `gateResult?.results ?? []`, so a Promise (no `.results`)
// resolves to an empty error list and the pre-flight gate stops blocking ANY release,
// forever, without a single visible failure. That is the exact "unmeasured reads as green"
// defect this order exists to fix, one level away — so this stays synchronous.
// Returns true/false only on a definitive HTTP response; null ("not checked") on any
// network failure — timeouts and DNS errors must not read as a confirmed dead link.
function checkHomepage(url) {
if (!url) return null;
try {
const out = execFileSync(
'curl', ['-s', '-o', '/dev/null', '-w', '%{http_code}', '--max-time', '5', '-L', url],
{ encoding: 'utf8' },
).trim();
const code = Number(out);
if (!Number.isFinite(code) || code === 0) return null;
return code < 400;
} catch {
return null;
}
}
// I/O shell — resolve one plugin's observed state from disk, then classify.
export function inspectPlugin(catalogDir, plugin) {
const name = plugin.name;
@ -230,10 +273,13 @@ export function inspectPlugin(catalogDir, plugin) {
}
let pluginVersion = null;
let homepage = null;
try {
const pj = JSON.parse(readFileSync(join(repoDir, '.claude-plugin', 'plugin.json'), 'utf8'));
pluginVersion = pj.version ?? null;
homepage = pj.homepage ?? null;
} catch { /* leave null → flagged */ }
const homepageReachable = checkHomepage(homepage);
let worktreeReadme = null;
try {
@ -255,22 +301,44 @@ export function inspectPlugin(catalogDir, plugin) {
catalogStats = extractCatalogStats(catalogReadme, name);
} catch { /* no catalog README → label + stat checks skipped */ }
return classifyPlugin({ name, catalogRef, pluginVersion, readmeBadge, tags: gitTags(repoDir), catalogLabel, statBadges, catalogStats });
return classifyPlugin({ name, catalogRef, pluginVersion, readmeBadge, tags: gitTags(repoDir), catalogLabel, statBadges, catalogStats, homepage, homepageReachable });
}
export function runGate(catalogDir, { strict = false } = {}) {
const mkt = JSON.parse(readFileSync(join(catalogDir, '.claude-plugin', 'marketplace.json'), 'utf8'));
const results = (mkt.plugins || []).map(p => inspectPlugin(catalogDir, p));
// Pure — takes classified results, not I/O. Split out of runGate so both directions of
// the SKIP-vs-green fix are unit-testable without needing a real sibling-repo layout on
// disk: a run that verified NOTHING
// (all SKIP) must fail even with zero ERROR/WARN, and a fully-verified clean run must
// still pass — a fix that only reddens the former without proving the latter would be
// unverified in the other direction.
export function gateOutcome(results, { strict = false, allowSkip = false } = {}) {
const total = results.length;
const skipped = results.filter(r => r.status === 'SKIP').length;
const hasError = results.some(r => r.status === 'ERROR');
const hasWarn = results.some(r => r.status === 'WARN');
return { results, hasError, hasWarn, failed: hasError || (strict && hasWarn) };
const unverified = skipped > 0 && !allowSkip;
return {
total,
skipped,
verified: total - skipped,
hasError,
hasWarn,
unverified,
failed: hasError || (strict && hasWarn) || unverified,
};
}
export function runGate(catalogDir, { strict = false, allowSkip = false } = {}) {
const mkt = JSON.parse(readFileSync(join(catalogDir, '.claude-plugin', 'marketplace.json'), 'utf8'));
const results = (mkt.plugins || []).map(p => inspectPlugin(catalogDir, p));
return { results, ...gateOutcome(results, { strict, allowSkip }) };
}
const __filename = fileURLToPath(import.meta.url);
if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) {
const strict = process.argv.includes('--strict');
const allowSkip = process.argv.includes('--allow-skip');
const catalogDir = join(dirname(__filename), '..');
const { results, failed } = runGate(catalogDir, { strict });
const { results, failed, skipped, total, verified } = runGate(catalogDir, { strict, allowSkip });
const icon = { OK: '✓', WARN: '⚠', ERROR: '✗', SKIP: '–' };
for (const r of results) {
@ -279,6 +347,10 @@ if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url)
}
const count = (s) => results.filter(r => r.status === s).length;
console.log('');
console.log(`${results.length} plugins — ${count('OK')} OK, ${count('WARN')} WARN, ${count('ERROR')} ERROR, ${count('SKIP')} SKIP${strict ? ' (strict)' : ''}`);
console.log(`${results.length} plugins — ${count('OK')} OK, ${count('WARN')} WARN, ${count('ERROR')} ERROR, ${count('SKIP')} SKIP${strict ? ' (strict)' : ''} — verified ${verified}/${total}`);
if (skipped > 0 && !allowSkip) {
console.log(`✗ ${skipped} plugin(s) UNVERIFIED (sibling repo not found locally) — this is NOT a green run.`);
console.log(` Clone the missing sibling repos next to this one, or pass --allow-skip to accept partial verification explicitly.`);
}
process.exit(failed ? 1 : 0);
}