Kjell Tore Guttormsen
0b0bf24137
check-versions.mjs reported "12 SKIP, exit 0" on a fresh clone with no sibling plugin repos — a run that verified nothing was indistinguishable from a clean run. Any SKIP now fails the gate by default (--allow-skip opts in explicitly), and the summary always reports "verified N/M". Also adds a homepage-resolves check on plugin.json (dead-link ERROR, verified live against voyage's stale pre-polyrepo URL); the field stays optional since 11 of 12 plugins don't carry it yet. gateOutcome is split out as a pure function so both directions (all-SKIP fails, fully-verified-clean still passes) are unit-tested without needing a real sibling-repo layout on disk.
401 lines
18 KiB
JavaScript
401 lines
18 KiB
JavaScript
// Tests for the marketplace version-consistency gate.
|
|
// Pure classifier is the unit under test — I/O shell (runGate/inspectPlugin) is exercised
|
|
// against the live tree by the CLI, not here.
|
|
import { test } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import {
|
|
normalizeVersion,
|
|
extractBadgeVersion,
|
|
extractCatalogLabel,
|
|
classifyPlugin,
|
|
extractStatBadges,
|
|
pickStatSource,
|
|
extractCatalogStats,
|
|
gateOutcome,
|
|
} from './check-versions.mjs';
|
|
|
|
test('normalizeVersion strips a leading v', () => {
|
|
assert.equal(normalizeVersion('v5.4.0'), '5.4.0');
|
|
assert.equal(normalizeVersion('5.4.0'), '5.4.0');
|
|
assert.equal(normalizeVersion(' v1.16.0 '), '1.16.0');
|
|
});
|
|
|
|
test('extractBadgeVersion pulls the version from a shields.io badge', () => {
|
|
assert.equal(extractBadgeVersion(''), '5.4.0');
|
|
assert.equal(extractBadgeVersion('no badge here'), null);
|
|
});
|
|
|
|
test('all-consistent plugin → OK', () => {
|
|
const r = classifyPlugin({
|
|
name: 'config-audit', catalogRef: 'v5.4.0', pluginVersion: '5.4.0',
|
|
readmeBadge: '5.4.0', tags: ['v5.4.0', 'v5.3.0'],
|
|
});
|
|
assert.equal(r.status, 'OK');
|
|
assert.ok(!r.findings.some(f => f.level === 'ERROR' || f.level === 'WARN'));
|
|
});
|
|
|
|
test('catalog ref with no matching tag → ERROR (install-breaking)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'voyage', catalogRef: 'v5.5.0', pluginVersion: '5.5.0',
|
|
readmeBadge: '5.5.0', tags: ['v5.1.1'],
|
|
});
|
|
assert.equal(r.status, 'ERROR');
|
|
assert.ok(r.findings.some(f => f.level === 'ERROR' && /no matching git tag/.test(f.msg)));
|
|
});
|
|
|
|
test('plugin.json version != README badge → ERROR (internal corruption)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'x', catalogRef: 'v1.0.0', pluginVersion: '1.0.0',
|
|
readmeBadge: '0.9.0', tags: ['v1.0.0'],
|
|
});
|
|
assert.equal(r.status, 'ERROR');
|
|
assert.ok(r.findings.some(f => f.level === 'ERROR' && /README version-badge/.test(f.msg)));
|
|
});
|
|
|
|
test('catalog ref behind a RELEASED version (tag exists) → WARN, suggests bumping catalog', () => {
|
|
const r = classifyPlugin({
|
|
name: 'x', catalogRef: 'v1.15.0', pluginVersion: '1.16.0',
|
|
readmeBadge: '1.16.0', tags: ['v1.16.0', 'v1.15.0'],
|
|
});
|
|
assert.equal(r.status, 'WARN');
|
|
assert.ok(r.findings.some(f => f.level === 'WARN' && /bump catalog ref/.test(f.msg)));
|
|
});
|
|
|
|
test('plugin.json ahead with no tag of its own → WARN, flags unreleased/untagged', () => {
|
|
const r = classifyPlugin({
|
|
name: 'ms-ai-architect', catalogRef: 'v1.15.0', pluginVersion: '1.16.0',
|
|
readmeBadge: '1.16.0', tags: ['v1.15.0'],
|
|
});
|
|
assert.equal(r.status, 'WARN');
|
|
assert.ok(r.findings.some(f => f.level === 'WARN' && /never tagged|unreleased/.test(f.msg)));
|
|
});
|
|
|
|
test('extractCatalogLabel reads the catalog README label by plugin name', () => {
|
|
const readme = [
|
|
'### [Config-Audit](https://git.fromaitochitta.com/open/config-audit) `v5.7.0`',
|
|
'### [MS AI Architect](https://git.fromaitochitta.com/open/ms-ai-architect) `v1.15.0` `🇳🇴 Norwegian`',
|
|
].join('\n');
|
|
assert.equal(extractCatalogLabel(readme, 'config-audit'), '5.7.0');
|
|
// first vX.Y.Z token wins — trailing flag/lang badge on the same line is ignored
|
|
assert.equal(extractCatalogLabel(readme, 'ms-ai-architect'), '1.15.0');
|
|
// no heading for this plugin → null (check is skipped)
|
|
assert.equal(extractCatalogLabel(readme, 'ghost'), null);
|
|
});
|
|
|
|
test('catalog README label != catalog ref → ERROR (doc misstates installed version)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'config-audit', catalogRef: 'v5.7.0', pluginVersion: '5.7.0',
|
|
readmeBadge: '5.7.0', tags: ['v5.7.0'], catalogLabel: '5.5.0',
|
|
});
|
|
assert.equal(r.status, 'ERROR');
|
|
assert.ok(r.findings.some(f => f.level === 'ERROR' && /README label/.test(f.msg)));
|
|
});
|
|
|
|
test('catalog README label == catalog ref → no label finding (stays OK)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'config-audit', catalogRef: 'v5.7.0', pluginVersion: '5.7.0',
|
|
readmeBadge: '5.7.0', tags: ['v5.7.0'], catalogLabel: '5.7.0',
|
|
});
|
|
assert.equal(r.status, 'OK');
|
|
});
|
|
|
|
test('label check does not fire on the legitimate ref-lags-plugin.json WARN case', () => {
|
|
// ref v1.15.0 lags plugin.json 1.16.0 (WARN), but the label matches the ref → no extra ERROR
|
|
const r = classifyPlugin({
|
|
name: 'ms-ai-architect', catalogRef: 'v1.15.0', pluginVersion: '1.16.0',
|
|
readmeBadge: '1.16.0', tags: ['v1.15.0'], catalogLabel: '1.15.0',
|
|
});
|
|
assert.equal(r.status, 'WARN');
|
|
assert.ok(!r.findings.some(f => f.level === 'ERROR'));
|
|
});
|
|
|
|
test('catalogLabel omitted (legacy callers) → label check skipped', () => {
|
|
const r = classifyPlugin({
|
|
name: 'x', catalogRef: 'v1.0.0', pluginVersion: '1.0.0',
|
|
readmeBadge: '1.0.0', tags: ['v1.0.0'],
|
|
});
|
|
assert.equal(r.status, 'OK');
|
|
});
|
|
|
|
test('plugin repo not found locally → SKIP', () => {
|
|
const r = classifyPlugin({
|
|
name: 'gone', catalogRef: 'v1.0.0', pluginVersion: null,
|
|
readmeBadge: null, tags: null,
|
|
});
|
|
assert.equal(r.status, 'SKIP');
|
|
});
|
|
|
|
test('ERROR dominates WARN when both apply', () => {
|
|
const r = classifyPlugin({
|
|
name: 'x', catalogRef: 'v2.0.0', pluginVersion: '2.1.0',
|
|
readmeBadge: '2.1.0', tags: ['v1.9.0'], // ref v2.0.0 dangling AND catalog behind v2.1.0
|
|
});
|
|
assert.equal(r.status, 'ERROR');
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stat-badge mirroring (added 2026-08-02, after measuring all 11 plugin READMEs
|
|
// + the catalog's 11 stat lines). Rule chosen by the operator: PER-AXIS. An axis
|
|
// is gated only when the plugin carries a matching badge; a badge-less axis is
|
|
// skipped, not flagged. Measured basis: 21 axis-pairs are badge-covered, and 14
|
|
// axes across 8 plugins are badge-less (voyage 4, ms-ai-architect 3,
|
|
// repo-mailbox 2, ai-psychosis 2, linkedin-studio 1, claude-design 1,
|
|
// graceful-handoff 1) — so a hardcoded per-PLUGIN exception list would have had
|
|
// to name 8 of 11 repos.
|
|
|
|
test('extractStatBadges reads numeric shields badges, ignoring non-numeric ones', () => {
|
|
const readme = [
|
|
'',
|
|
'',
|
|
'',
|
|
'',
|
|
'',
|
|
].join('\n');
|
|
const s = extractStatBadges(readme);
|
|
assert.equal(s.get('command'), 20);
|
|
assert.equal(s.get('test'), 2013);
|
|
// version/platform/license carry non-numeric values → never stat axes
|
|
assert.equal(s.has('version'), false);
|
|
assert.equal(s.has('platform'), false);
|
|
assert.equal(s.has('license'), false);
|
|
});
|
|
|
|
test('extractStatBadges handles a zero value and the -- literal-hyphen escape', () => {
|
|
const readme = [
|
|
'',
|
|
'',
|
|
].join('\n');
|
|
const s = extractStatBadges(readme);
|
|
assert.equal(s.get('hook'), 0, '0 is a real count, not a missing badge');
|
|
assert.equal(s.has('state-helper'), false, 'non-numeric value → not a stat axis');
|
|
});
|
|
|
|
test('extractCatalogStats parses the per-plugin stat line into axis counts', () => {
|
|
const cat = [
|
|
'### [Config-Audit](https://git.fromaitochitta.com/open/config-audit) `v5.13.0`',
|
|
'',
|
|
'Some prose with 99 red herrings in it.',
|
|
'',
|
|
'7 agents · 16 scanners · 21 commands · 1410 tests · [Full documentation →](https://x)',
|
|
'',
|
|
'---',
|
|
].join('\n');
|
|
const s = extractCatalogStats(cat, 'config-audit');
|
|
assert.equal(s.get('agent'), 7);
|
|
assert.equal(s.get('scanner'), 16);
|
|
assert.equal(s.get('command'), 21);
|
|
assert.equal(s.get('test'), 1410);
|
|
assert.equal(s.has('red herring'), false, 'prose above the stat line must not leak in');
|
|
});
|
|
|
|
test('extractCatalogStats reads a parenthetical count as its own axis', () => {
|
|
const cat = [
|
|
'### [MS AI Architect](https://git.fromaitochitta.com/open/ms-ai-architect) `v1.17.0`',
|
|
'',
|
|
'12 agents · 29 commands · 5 skills (389 docs) · 2 hooks · [Full documentation →](https://x)',
|
|
].join('\n');
|
|
const s = extractCatalogStats(cat, 'ms-ai-architect');
|
|
assert.equal(s.get('skill'), 5);
|
|
assert.equal(s.get('reference'), 389, 'doc/docs normalizes onto the reference axis');
|
|
assert.equal(s.get('hook'), 2);
|
|
});
|
|
|
|
test('extractCatalogStats returns an empty map when the plugin has no entry', () => {
|
|
assert.equal(extractCatalogStats('### [Other](https://x/open/other) `v1.0.0`', 'absent').size, 0);
|
|
});
|
|
|
|
test('stat mismatch on a badge-covered axis → ERROR (the config-audit 1441/1410 case)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'config-audit', catalogRef: 'v5.13.0', pluginVersion: '5.13.0',
|
|
readmeBadge: '5.13.0', tags: ['v5.13.0'], catalogLabel: '5.13.0',
|
|
statBadges: new Map([['test', 1441], ['agent', 7]]),
|
|
catalogStats: new Map([['test', 1410], ['agent', 7]]),
|
|
});
|
|
assert.equal(r.status, 'ERROR');
|
|
assert.ok(r.findings.some(f => f.level === 'ERROR' && /test/.test(f.msg) && /1441/.test(f.msg) && /1410/.test(f.msg)));
|
|
});
|
|
|
|
test('every stat axis agreeing → stays OK', () => {
|
|
const r = classifyPlugin({
|
|
name: 'okr', catalogRef: 'v1.8.2', pluginVersion: '1.8.2',
|
|
readmeBadge: '1.8.2', tags: ['v1.8.2'], catalogLabel: '1.8.2',
|
|
statBadges: new Map([['agent', 7], ['command', 14], ['hook', 3], ['reference', 17]]),
|
|
catalogStats: new Map([['agent', 7], ['command', 14], ['hook', 3]]),
|
|
});
|
|
assert.equal(r.status, 'OK', 'a badge the catalog simply does not restate is not a finding');
|
|
});
|
|
|
|
test('badge-less catalog axis is SKIPPED, not flagged (the whole voyage case)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'voyage', catalogRef: 'v5.9.1', pluginVersion: '5.9.1',
|
|
readmeBadge: '5.9.1', tags: ['v5.9.1'], catalogLabel: '5.9.1',
|
|
statBadges: new Map(),
|
|
catalogStats: new Map([['agent', 24], ['command', 6], ['hook', 7], ['test', 500]]),
|
|
});
|
|
assert.equal(r.status, 'OK');
|
|
assert.ok(!r.findings.some(f => f.level === 'ERROR'), 'no badge → no claim to mirror → no error');
|
|
});
|
|
|
|
test('a zero badge still gates (0 != 3 is a real mismatch, not a missing badge)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
|
tags: ['v1.0.0'], catalogLabel: '1.0.0',
|
|
statBadges: new Map([['hook', 0]]),
|
|
catalogStats: new Map([['hook', 3]]),
|
|
});
|
|
assert.equal(r.status, 'ERROR');
|
|
});
|
|
|
|
test('an approximate catalog count (N+) gates as a LOWER BOUND', () => {
|
|
const under = classifyPlugin({
|
|
name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
|
tags: ['v1.0.0'], catalogLabel: '1.0.0',
|
|
statBadges: new Map([['test', 400]]),
|
|
catalogStats: new Map([['test', { atLeast: 500 }]]),
|
|
});
|
|
assert.equal(under.status, 'ERROR', '"500+ tests" while the badge says 400 overstates');
|
|
|
|
const over = classifyPlugin({
|
|
name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
|
tags: ['v1.0.0'], catalogLabel: '1.0.0',
|
|
statBadges: new Map([['test', 2013]]),
|
|
catalogStats: new Map([['test', { atLeast: 500 }]]),
|
|
});
|
|
assert.equal(over.status, 'OK', '"500+" is satisfied by any badge >= 500');
|
|
});
|
|
|
|
test('stat maps omitted (legacy callers) → mirroring skipped entirely', () => {
|
|
const r = classifyPlugin({
|
|
name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0',
|
|
readmeBadge: '1.0.0', tags: ['v1.0.0'], catalogLabel: '1.0.0',
|
|
});
|
|
assert.equal(r.status, 'OK');
|
|
});
|
|
|
|
// Regression, found live 2026-08-02: llm-security had committed past its v7.8.3 tag
|
|
// (scanners 23->22, tests 2013->2034) WITHOUT bumping the version. The gate read the
|
|
// sibling's working tree and reported the catalog stale — but the catalog documents
|
|
// what INSTALLS, and `ref: v7.8.3` still installs 23/2013. The catalog was right and
|
|
// the gate was wrong. Stat badges must therefore be read at the pinned ref.
|
|
test('pickStatSource prefers the README at the pinned ref over the working tree', () => {
|
|
const atRef = 'https://img.shields.io/badge/scanners-23-cyan';
|
|
const atWorktree = 'https://img.shields.io/badge/scanners-22-cyan';
|
|
assert.equal(pickStatSource({ atRef, atWorktree }), atRef);
|
|
assert.equal(extractStatBadges(pickStatSource({ atRef, atWorktree })).get('scanner'), 23);
|
|
});
|
|
|
|
test('pickStatSource falls back to the working tree only when the ref cannot be read', () => {
|
|
const atWorktree = 'https://img.shields.io/badge/scanners-22-cyan';
|
|
assert.equal(pickStatSource({ atRef: null, atWorktree }), atWorktree);
|
|
assert.equal(pickStatSource({ atRef: null, atWorktree: null }), null);
|
|
});
|
|
|
|
test('an empty README at the ref is a real read, not a failed one', () => {
|
|
assert.equal(pickStatSource({ atRef: '', atWorktree: 'badge/scanners-22-cyan' }), '');
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// gateOutcome — the fix for the UNMEASURED-reads-as-green bug found 2026-08-18.
|
|
// A fresh clone with no sibling repos cloned produced "12 plugins — 0 OK, 0
|
|
// WARN, 0 ERROR, 12 SKIP, exit 0": a run that verified NOTHING was
|
|
// indistinguishable from a run that verified everything and found it clean.
|
|
// Both directions must hold, or a fix that only
|
|
// reddens the all-SKIP case could just as easily redden (or silently pass) the
|
|
// all-verified case too.
|
|
|
|
test('gateOutcome: all-SKIP (nothing verified) fails even with zero ERROR/WARN', () => {
|
|
const results = [{ status: 'SKIP' }, { status: 'SKIP' }, { status: 'SKIP' }];
|
|
const o = gateOutcome(results);
|
|
assert.equal(o.hasError, false);
|
|
assert.equal(o.skipped, 3);
|
|
assert.equal(o.verified, 0);
|
|
assert.equal(o.total, 3);
|
|
assert.equal(o.failed, true, 'a run that verified nothing must not report green');
|
|
});
|
|
|
|
test('gateOutcome: fully-verified all-OK run stays green (the fix must not break the normal case)', () => {
|
|
const results = [{ status: 'OK' }, { status: 'OK' }, { status: 'WARN' }];
|
|
const o = gateOutcome(results);
|
|
assert.equal(o.skipped, 0);
|
|
assert.equal(o.verified, 3);
|
|
assert.equal(o.failed, false, 'no ERROR, no SKIP, non-strict → still green');
|
|
});
|
|
|
|
test('gateOutcome: ERROR still fails regardless of skip count', () => {
|
|
const o = gateOutcome([{ status: 'ERROR' }, { status: 'SKIP' }]);
|
|
assert.equal(o.failed, true);
|
|
});
|
|
|
|
test('gateOutcome: --allow-skip lets a partially-unverified run pass deliberately', () => {
|
|
const results = [{ status: 'OK' }, { status: 'SKIP' }];
|
|
const withoutFlag = gateOutcome(results);
|
|
const withFlag = gateOutcome(results, { allowSkip: true });
|
|
assert.equal(withoutFlag.failed, true, 'default: unverified plugin blocks green');
|
|
assert.equal(withFlag.failed, false, 'explicit opt-in: caller accepts partial verification');
|
|
});
|
|
|
|
test('gateOutcome: strict mode still fails on WARN even with everything verified', () => {
|
|
const o = gateOutcome([{ status: 'WARN' }], { strict: true });
|
|
assert.equal(o.failed, true);
|
|
});
|
|
|
|
test('gateOutcome: verified/total denominator is reported for the summary line', () => {
|
|
const o = gateOutcome([{ status: 'OK' }, { status: 'OK' }, { status: 'SKIP' }]);
|
|
assert.equal(o.total, 3);
|
|
assert.equal(o.verified, 2);
|
|
assert.equal(o.skipped, 1);
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Manifest `homepage` must resolve when present (order finding 2 — voyage's
|
|
// manifest pointed at a pre-polyrepo path that 404s). The field stays OPTIONAL
|
|
// (11 of 12 plugins omit it) — this is a deliberate policy call made in this
|
|
// session, not an oversight: making it required would flip 11 plugins to ERROR
|
|
// and, via release-plugin.mjs's ERROR-set pre-flight, block every release in
|
|
// the marketplace until all 11 sibling repos re-tag. See check-versions.mjs
|
|
// for the same note at the check site.
|
|
|
|
test('homepage present and reachable → no finding (known-positive)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'voyage', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
|
tags: ['v1.0.0'], homepage: 'https://git.fromaitochitta.com/open/voyage', homepageReachable: true,
|
|
});
|
|
assert.equal(r.status, 'OK');
|
|
assert.ok(!r.findings.some(f => /homepage/.test(f.msg)));
|
|
});
|
|
|
|
test('homepage present and dead → ERROR (known-negative, the voyage 404 case)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'voyage', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
|
tags: ['v1.0.0'],
|
|
homepage: 'https://git.fromaitochitta.com/open/ktg-plugin-marketplace/src/branch/main/plugins/voyage',
|
|
homepageReachable: false,
|
|
});
|
|
assert.equal(r.status, 'ERROR');
|
|
assert.ok(r.findings.some(f => f.level === 'ERROR' && /homepage/.test(f.msg) && /does not resolve/.test(f.msg)));
|
|
});
|
|
|
|
test('homepage absent → no finding (field is optional, not every plugin must carry it)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'config-audit', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
|
tags: ['v1.0.0'], homepage: null, homepageReachable: null,
|
|
});
|
|
assert.equal(r.status, 'OK');
|
|
assert.ok(!r.findings.some(f => /homepage/.test(f.msg)));
|
|
});
|
|
|
|
test('homepage present but unchecked (network unreachable) → not silently treated as reachable, but does not false-fail', () => {
|
|
const r = classifyPlugin({
|
|
name: 'voyage', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
|
tags: ['v1.0.0'], homepage: 'https://git.fromaitochitta.com/open/voyage', homepageReachable: null,
|
|
});
|
|
assert.equal(r.status, 'OK', 'unmeasured must not be reported as a confirmed dead link');
|
|
assert.ok(!r.findings.some(f => /homepage/.test(f.msg)));
|
|
});
|
|
|
|
test('homepage omitted entirely (legacy callers) → check skipped, stays OK', () => {
|
|
const r = classifyPlugin({
|
|
name: 'x', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0', tags: ['v1.0.0'],
|
|
});
|
|
assert.equal(r.status, 'OK');
|
|
});
|