ktg-plugin-marketplace/scripts/check-versions.mjs
Kjell Tore Guttormsen af4a3cce32
feat(check-versions): name ungated stat axes instead of skipping them silently
A stat-line axis the plugin does not badge was skipped in silence, so
"checked and right" and "never checked" printed the same thing. Each
plugin now gets an INFO line naming its ungated axes, and the run ends
with "stat claims - N gated - M NOT gated (no badge)". INFO never changes
status or exit code, not even under --strict. Suggested by repo-standard
(reply on BADGE-COUNT, 2026-11-03); operator decision 2026-11-04.

First live run: voyage 0 gated / 5 not gated. Suite 244/244.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-04 09:33:52 +01:00

419 lines
21 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env node
// Marketplace version-consistency gate.
//
// For every plugin in catalog/.claude-plugin/marketplace.json, compare its catalog `ref`
// against the plugin repo found as a sibling directory (../<name>):
// - catalog ref must point at an existing git tag → else ERROR (install-breaking)
// - plugin.json version must equal the README version-badge → else ERROR (internal)
// - catalog README per-plugin label must equal catalog ref → else ERROR (doc misstates install)
// - catalog ref should equal plugin.json version → else WARN (catalog lags / unreleased bump)
// - plugin.json homepage, if present, must resolve → else ERROR (dead link); field is OPTIONAL
// - sibling repo missing → SKIP
//
// SKIP is UNVERIFIED, not clean — a plugin that could not be checked at all is not the
// same thing as a plugin that was checked and found fine. A run with any SKIP therefore
// exits 1 by default (fixes: 2026-08-18 a fresh clone with zero sibling repos printed
// "12 SKIP, exit 0" — a run that verified NOTHING read as green). Pass --allow-skip to
// accept partial verification explicitly; the summary line always reports "verified N/M".
//
// Exit 1 if any ERROR, any SKIP (unless --allow-skip), or any WARN under --strict; 0 otherwise.
// Usage: node scripts/check-versions.mjs [--strict] [--allow-skip]
import { readFileSync, existsSync } from 'node:fs';
import { execFileSync } from 'node:child_process';
import { join, dirname } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
export function normalizeVersion(v) {
return String(v).trim().replace(/^v/, '');
}
export function extractBadgeVersion(readmeText) {
const m = /badge\/version-(\d+\.\d+\.\d+)/.exec(readmeText || '');
return m ? m[1] : null;
}
// Read the catalog README's per-plugin version label, e.g.
// ### [Config-Audit](https://.../open/config-audit) `v5.7.0`
// Matches the heading line by its `/open/<name>)` link and returns the FIRST `vX.Y.Z`
// token on it (so a trailing `🇳🇴 Norwegian`-style badge is ignored). null if no entry.
export function extractCatalogLabel(readmeText, name) {
for (const line of String(readmeText || '').split('\n')) {
if (line.includes(`/open/${name})`)) {
const m = /`v(\d+\.\d+\.\d+)`/.exec(line);
return m ? m[1] : null;
}
}
return null;
}
// --- stat-badge mirroring -------------------------------------------------
//
// The catalog restates each plugin's counts in a per-plugin stat line
// ("7 agents · 16 scanners · 21 commands · 1410 tests · [Full documentation →]").
// Those numbers rot: measured 2026-08-02, config-audit's line said 1410 tests
// while the plugin's own badge said 1441. This mirrors catalog line -> plugin badge.
//
// The rule is PER-AXIS, not per-plugin. An axis is gated only when the plugin
// carries a badge for it; a badge-less axis is not checked (but named — see statCoverage). That is not a
// concession to voyage (the one plugin with no stat badges at all) — re-measured
// 2026-08-04 at the pinned refs, 15 of the 42 axis-claims across 7 of the 12
// plugins are badge-less, so a per-plugin exception list would have had to name
// most of the marketplace and be edited by hand forever. The cost is stated
// plainly: those 15 numbers stay ungated and need a human pass whenever a ref
// moves — repo-mailbox's `6 CLI scripts` was 8 when measured, and it badges both
// of its axes now, so they are gated as of v0.20.2. (Its selftest-check figure is
// the standing proof that ungated numbers rot: the catalog said 251 while three
// records of the true value disagree — see CLAUDE.md. Not re-measured.)
const AXIS_SYNONYM = { 'reference doc': 'reference', 'knowledge doc': 'reference', doc: 'reference' };
// Fold a badge label or a stat-line noun onto a shared axis name.
export function normalizeAxis(raw) {
let s = String(raw || '').replace(/--/g, '-').replace(/%20/g, ' ').replace(/_/g, ' ')
.toLowerCase().replace(/\s+/g, ' ').trim();
s = s.replace(/\s+files?$/, '');
const words = s.split(' ');
const last = words[words.length - 1];
if (last.length > 3 && last.endsWith('s') && !last.endsWith('ss')) words[words.length - 1] = last.slice(0, -1);
s = words.join(' ');
return AXIS_SYNONYM[s] || s;
}
// Plugin README -> Map(axis -> count). Only badges whose VALUE is an integer are
// stat axes; version/platform/license carry text and are ignored. Shields escapes
// a literal hyphen as `--`, so the label/value/color split must ignore doubled ones.
export function extractStatBadges(readmeText) {
const out = new Map();
for (const m of String(readmeText || '').matchAll(/img\.shields\.io\/badge\/([^)\s]+)/g)) {
const parts = m[1].split(/(?<!-)-(?!-)/);
if (parts.length < 3) continue;
if (!/^\d+$/.test(parts[1])) continue;
out.set(normalizeAxis(parts[0]), Number(parts[1]));
}
return out;
}
// Pull "<n> <axis>" pairs out of one `·`-separated segment, including any
// parenthetical aside ("5 skills (389 docs)" is two axes; "(+1 helper)" is one).
function readStatSegment(segment, out) {
const asides = [...segment.matchAll(/\(([^)]*)\)/g)].map(a => a[1]);
const main = segment.replace(/\([^)]*\)/g, ' ');
for (const piece of [main, ...asides]) {
const m = /(\d+)(\+?)\s+([A-Za-z][A-Za-z -]*)/.exec(piece);
if (!m) continue;
const axis = normalizeAxis(m[3]);
if (!axis || out.has(axis)) continue;
out.set(axis, m[2] === '+' ? { atLeast: Number(m[1]) } : Number(m[1]));
}
}
// Which copy of the plugin README the stat mirror must read.
//
// The catalog documents what INSTALLS, and what installs is the pinned `ref` — not
// whatever the sibling working tree happens to hold. Measured live 2026-08-02:
// llm-security had committed past its v7.8.3 tag (scanners 23->22, tests 2013->2034)
// without bumping the version, so reading the working tree reported the catalog stale
// while the catalog was correct about v7.8.3. Fall back to the working tree only when
// the ref cannot be read at all — a ref that resolves to nothing is already its own
// ERROR (check 1), so this fallback never hides a dangling ref.
//
// Reading the ref decides COVERAGE too, not just values. Measured 2026-08-04:
// graceful-handoff had dropped its `tests-30` and `hooks-0` badges on main (13e2972,
// 5e17409) without releasing, so the working tree says the catalog's `30 tests` is
// ungated while at v3.1.0 it is badged and gated. The failure directions are opposite
// — the llm-security case above makes a correct catalog look stale — so neither one
// alone would have caught this.
export function pickStatSource({ atRef, atWorktree }) {
return atRef !== null && atRef !== undefined ? atRef : (atWorktree ?? null);
}
// Catalog README -> Map(axis -> count | {atLeast}). Reads ONLY the stat line of the
// named plugin's block, so prose numbers above it can never be mistaken for counts.
export function extractCatalogStats(readmeText, name) {
const lines = String(readmeText || '').split('\n');
const out = new Map();
let inBlock = false;
for (const line of lines) {
if (line.startsWith('### [')) inBlock = line.includes(`/open/${name})`);
if (!inBlock || !line.includes('[Full documentation')) continue;
for (const seg of line.split('[Full documentation')[0].split('·')) readStatSegment(seg, out);
}
return out;
}
// Compare a catalog's stated stat-line counts for one plugin against that plugin's own
// stat badges — per-axis, badge-gated (see the comment block above extractStatBadges).
// Pure, and shared by two callers with different timing: classifyPlugin (post-hoc, gates
// a release that has already happened) and release-plugin.mjs's pre-release check
// (Q3e/D1 — BEFORE anything is tagged or written, order 20260913T051659Z-717911204).
export function statMismatchFindings(catalogStats, statBadges) {
const findings = [];
if (!catalogStats || !statBadges) return findings;
for (const [axis, stated] of catalogStats) {
if (!statBadges.has(axis)) continue;
const badge = statBadges.get(axis);
const approx = stated !== null && typeof stated === 'object';
const bad = approx ? badge < stated.atLeast : badge !== stated;
if (bad) {
const shown = approx ? `${stated.atLeast}+` : String(stated);
findings.push({
level: 'ERROR',
// `kind` is a machine-readable discriminator, not decoration: release-plugin.mjs's
// pre-flight must tell a stat finding from a dangling ref WITHOUT matching on prose.
kind: 'stat',
msg: `catalog says ${shown} ${axis} but the plugin's badge says ${badge} (catalog stat line is stale)`,
});
}
}
return findings;
}
// Which of the catalog's stated axes the gate can actually check (operator decision
// 2026-11-04, after repo-standard's reply on BADGE-COUNT). Skipping a badge-less axis
// in silence made "checked and right" and "never checked" print the same thing — the
// SKIP-is-not-a-pass rule above, one level up. Every badge trim (repo-standard's
// BADGE-COUNT pushes plugins to trim) moves claims from `gated` to `ungated`, so the
// leak has to be visible in the gate's own output, whoever does the trimming.
export function statCoverage(catalogStats, statBadges) {
const gated = [];
const ungated = [];
if (!catalogStats || !statBadges) return { gated, ungated };
for (const axis of catalogStats.keys()) (statBadges.has(axis) ? gated : ungated).push(axis);
return { gated, ungated };
}
// Catalog-wide denominator for the summary line: how many stated stat claims are gated,
// how many are not, and which plugins carry the ungated ones. Reads the `ungated` INFO
// finding classifyPlugin emits (and its `gated` count), so it counts what was reported.
export function coverageSummary(results) {
let gated = 0;
let ungated = 0;
const plugins = [];
for (const r of results) {
const cov = (r.findings ?? []).find(f => f.kind === 'coverage');
if (!cov) continue;
gated += cov.gated.length;
ungated += cov.axes.length;
if (cov.axes.length > 0) plugins.push(r.name);
}
return { gated, ungated, plugins };
}
// Pure classifier — all I/O is resolved into the input shape before this is called.
// tags === null means "repo not inspected" (missing locally); [] means "no tags".
export function classifyPlugin({ name, catalogRef, pluginVersion, readmeBadge, tags, catalogLabel = null, statBadges = null, catalogStats = null, homepage = null, homepageReachable = null }) {
if (pluginVersion === null && tags === null) {
return { name, status: 'SKIP', findings: [{ level: 'SKIP', msg: 'plugin repo not found locally — cannot verify' }] };
}
const findings = [];
if (pluginVersion === null) {
findings.push({ level: 'ERROR', msg: 'plugin.json version not found or unparseable' });
}
// 1. dangling ref — catalog points at a tag that does not exist in the plugin repo
if (tags !== null && !tags.includes(catalogRef)) {
findings.push({ level: 'ERROR', msg: `catalog ref ${catalogRef} has no matching git tag in the plugin repo (install-breaking)` });
}
// 2. internal consistency — plugin.json version vs README version-badge
if (pluginVersion !== null && readmeBadge !== null && pluginVersion !== readmeBadge) {
findings.push({ level: 'ERROR', msg: `plugin.json version ${pluginVersion} != README version-badge ${readmeBadge}` });
}
// 3. catalog README label vs catalog ref — the human-facing doc must match what installs.
// Unlike ref-vs-plugin.json, there is no legitimate transient state where these differ.
if (catalogLabel !== null && normalizeVersion(catalogRef) !== catalogLabel) {
findings.push({ level: 'ERROR', msg: `catalog README label v${catalogLabel} != catalog ref ${catalogRef} (README misstates the installed version)` });
}
// 4. catalog ref vs plugin.json version
if (pluginVersion !== null && normalizeVersion(catalogRef) !== pluginVersion) {
const releasedTag = tags !== null && tags.includes('v' + pluginVersion);
const reason = releasedTag
? `released version v${pluginVersion} exists as a tag — bump catalog ref`
: `no tag for v${pluginVersion} — unreleased bump, or a release that was never tagged`;
findings.push({ level: 'WARN', msg: `catalog ref ${catalogRef} != plugin.json version ${pluginVersion} (${reason})` });
}
// 5. catalog stat line vs the plugin's own stat badges — per-axis, badge-gated.
// An axis the plugin does not badge is SKIPPED (no claim to mirror). An axis the
// badge carries but the catalog does not restate is likewise nothing: the catalog
// chooses what to show. Only a stated number contradicting a badged one is a defect.
if (statBadges !== null && catalogStats !== null) {
findings.push(...statMismatchFindings(catalogStats, statBadges));
}
// 5b. ...and the axes step 5 could NOT check are named, never skipped in silence.
// INFO is outside the status computation below: a stated number with no badge is
// legitimate, it is just unverified, and it needs a human pass whenever the ref moves.
const coverage = statCoverage(catalogStats, statBadges);
// 6. plugin.json homepage must resolve when present — the first link an agent follows
// from a manifest. The field stays OPTIONAL (measured 2026-08-18: 11 of 12 plugins omit
// it) — a deliberate policy call, not an oversight. Making it required would flip those
// 11 to ERROR and, via release-plugin.mjs's ERROR-set pre-flight (catalog-wide), block
// every release in the marketplace until each of those 11 sibling repos re-tags. That is
// a marketplace-wide freeze delivered by a script fix, not a gate improvement.
// `homepageReachable === null` means "not checked" (e.g. network unreachable) and must
// NOT read as either pass or fail.
if (homepage !== null && homepageReachable === false) {
findings.push({ level: 'ERROR', msg: `plugin.json homepage ${homepage} does not resolve (dead link)` });
}
const status = findings.some(f => f.level === 'ERROR') ? 'ERROR'
: findings.some(f => f.level === 'WARN') ? 'WARN'
: 'OK';
if (findings.length === 0) findings.push({ level: 'OK', msg: `consistent at ${pluginVersion}` });
if (coverage.gated.length + coverage.ungated.length > 0) {
findings.push({
level: coverage.ungated.length > 0 ? 'INFO' : 'OK',
kind: 'coverage',
axes: coverage.ungated,
gated: coverage.gated,
msg: coverage.ungated.length > 0
? `${coverage.ungated.join(', ')}: not gated — no badge (re-measure at the tag when the ref moves)`
: `stat line fully gated (${coverage.gated.join(', ')})`,
});
}
return { name, status, findings };
}
// Read one file as it exists AT a git ref, without touching the working tree.
// null when the ref or the path does not resolve there.
function gitShow(repoDir, ref, path) {
if (!ref) return null;
try {
return execFileSync('git', ['-C', repoDir, 'show', `${ref}:${path}`], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] });
} catch {
return null;
}
}
function gitTags(repoDir) {
try {
const out = execFileSync('git', ['-C', repoDir, 'tag', '--list', 'v*'], { encoding: 'utf8' });
return out.split('\n').map(s => s.trim()).filter(Boolean);
} catch {
return null;
}
}
// Synchronous on purpose (curl via execFileSync, not fetch): inspectPlugin/runGate must
// stay synchronous because release-plugin.mjs's applyRelease calls `io.runGate(catalogDir)`
// without awaiting it. Turning runGate async would make that call site silently receive a
// Promise — preflightErrors reads `gateResult?.results ?? []`, so a Promise (no `.results`)
// resolves to an empty error list and the pre-flight gate stops blocking ANY release,
// forever, without a single visible failure. That is the exact "unmeasured reads as green"
// defect this order exists to fix, one level away — so this stays synchronous.
// Returns true/false only on a definitive HTTP response; null ("not checked") on any
// network failure — timeouts and DNS errors must not read as a confirmed dead link.
function checkHomepage(url) {
if (!url) return null;
try {
const out = execFileSync(
'curl', ['-s', '-o', '/dev/null', '-w', '%{http_code}', '--max-time', '5', '-L', url],
{ encoding: 'utf8' },
).trim();
const code = Number(out);
if (!Number.isFinite(code) || code === 0) return null;
return code < 400;
} catch {
return null;
}
}
// I/O shell — resolve one plugin's observed state from disk, then classify.
export function inspectPlugin(catalogDir, plugin) {
const name = plugin.name;
const catalogRef = plugin.source?.ref ?? null;
const repoDir = join(catalogDir, '..', name);
if (!existsSync(repoDir)) {
return classifyPlugin({ name, catalogRef, pluginVersion: null, readmeBadge: null, tags: null });
}
let pluginVersion = null;
let homepage = null;
try {
const pj = JSON.parse(readFileSync(join(repoDir, '.claude-plugin', 'plugin.json'), 'utf8'));
pluginVersion = pj.version ?? null;
homepage = pj.homepage ?? null;
} catch { /* leave null → flagged */ }
const homepageReachable = checkHomepage(homepage);
let worktreeReadme = null;
try {
worktreeReadme = readFileSync(join(repoDir, 'README.md'), 'utf8');
} catch { /* no README → badge + stat checks skipped */ }
// Version-badge check 2 is about the working tree's INTERNAL consistency, so it keeps
// reading the working tree. The stat mirror is about what the catalog promises users,
// so it reads the pinned ref instead — see pickStatSource.
const readmeBadge = worktreeReadme === null ? null : extractBadgeVersion(worktreeReadme);
const statSource = pickStatSource({ atRef: gitShow(repoDir, catalogRef, 'README.md'), atWorktree: worktreeReadme });
const statBadges = statSource === null ? null : extractStatBadges(statSource);
let catalogLabel = null;
let catalogStats = null;
try {
const catalogReadme = readFileSync(join(catalogDir, 'README.md'), 'utf8');
catalogLabel = extractCatalogLabel(catalogReadme, name);
catalogStats = extractCatalogStats(catalogReadme, name);
} catch { /* no catalog README → label + stat checks skipped */ }
return classifyPlugin({ name, catalogRef, pluginVersion, readmeBadge, tags: gitTags(repoDir), catalogLabel, statBadges, catalogStats, homepage, homepageReachable });
}
// Pure — takes classified results, not I/O. Split out of runGate so both directions of
// the SKIP-vs-green fix are unit-testable without needing a real sibling-repo layout on
// disk: a run that verified NOTHING
// (all SKIP) must fail even with zero ERROR/WARN, and a fully-verified clean run must
// still pass — a fix that only reddens the former without proving the latter would be
// unverified in the other direction.
export function gateOutcome(results, { strict = false, allowSkip = false } = {}) {
const total = results.length;
const skipped = results.filter(r => r.status === 'SKIP').length;
const hasError = results.some(r => r.status === 'ERROR');
const hasWarn = results.some(r => r.status === 'WARN');
const unverified = skipped > 0 && !allowSkip;
return {
total,
skipped,
verified: total - skipped,
hasError,
hasWarn,
unverified,
failed: hasError || (strict && hasWarn) || unverified,
};
}
export function runGate(catalogDir, { strict = false, allowSkip = false } = {}) {
const mkt = JSON.parse(readFileSync(join(catalogDir, '.claude-plugin', 'marketplace.json'), 'utf8'));
const results = (mkt.plugins || []).map(p => inspectPlugin(catalogDir, p));
return { results, ...gateOutcome(results, { strict, allowSkip }) };
}
const __filename = fileURLToPath(import.meta.url);
if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) {
const strict = process.argv.includes('--strict');
const allowSkip = process.argv.includes('--allow-skip');
const catalogDir = join(dirname(__filename), '..');
const { results, failed, skipped, total, verified } = runGate(catalogDir, { strict, allowSkip });
const icon = { OK: '✓', WARN: '⚠', ERROR: '✗', SKIP: '–' };
for (const r of results) {
console.log(`${icon[r.status] || '?'} ${r.status.padEnd(5)} ${r.name}`);
for (const f of r.findings) if (f.level !== 'OK') console.log(` ${f.msg}`);
}
const count = (s) => results.filter(r => r.status === s).length;
const cov = coverageSummary(results);
console.log('');
console.log(`stat claims — ${cov.gated} gated · ${cov.ungated} NOT gated (no badge)${cov.plugins.length ? ` in ${cov.plugins.join(', ')}` : ''}`);
console.log(`${results.length} plugins — ${count('OK')} OK, ${count('WARN')} WARN, ${count('ERROR')} ERROR, ${count('SKIP')} SKIP${strict ? ' (strict)' : ''} — verified ${verified}/${total}`);
if (skipped > 0 && !allowSkip) {
console.log(`✗ ${skipped} plugin(s) UNVERIFIED (sibling repo not found locally) — this is NOT a green run.`);
console.log(` Clone the missing sibling repos next to this one, or pass --allow-skip to accept partial verification explicitly.`);
}
process.exit(failed ? 1 : 0);
}