Kjell Tore Guttormsen
fc95cbd300
[skip-docs]: the user-facing doc here is CLAUDE.md, not README.md. This is a
maintainer-side consistency gate — the catalog landing page should not describe
it — and CLAUDE.md gains a full paragraph covering the rule, the measured
badge-less gap, and the never-hand-edit-a-stat-line instruction. README.md
changes by exactly one number because the gate found it wrong.
The catalog restates each plugin's counts in a per-plugin stat line, and those
numbers rot silently: nothing compared them to anything. Measured across all 11
plugin READMEs today, config-audit's line claimed 1410 tests while the plugin's
own badge said 1441.
Rule is PER-AXIS, not per-plugin. For each number on a catalog stat line, if the
plugin carries a shields badge for that axis, they must agree (ERROR otherwise);
if it carries no such badge, the axis is skipped. This was measured, not assumed:
21 axis-pairs are badge-covered, but 14 axes across 8 of the 11 plugins are
badge-less (voyage 4, ms-ai-architect 3, repo-mailbox 2, ai-psychosis 2,
linkedin-studio 1, claude-design 1, graceful-handoff 1). A per-plugin exception
list — the original sketch, scoped around voyage alone — would have had to name 8
of 11 repos and be hand-edited for every new axis.
Stated plainly rather than hidden: those 14 axes stay ungated. repo-mailbox's two
were 6 and 251 against a true 8 and 374 (fixed in e9054f8).
Parsing is measured against the real corpus: shields' `--` literal-hyphen escape,
`_`/`%20` spaces, zero as a real count, non-numeric badges (version/platform/
license) excluded, parenthetical asides read as their own axis ("5 skills (389
docs)"), doc/docs/references/reference_docs/knowledge_docs folded onto one axis,
and `N+` read as a lower bound.
Only the catalog's config-audit number changed; the plugin badge is the source.
Tests 106 -> 117 (+11). Gate green at 11 OK / 0 WARN / 0 ERROR.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RDSiMcgLMpEETwtkc86Nym
271 lines
11 KiB
JavaScript
271 lines
11 KiB
JavaScript
// Tests for the marketplace version-consistency gate.
|
|
// Pure classifier is the unit under test — I/O shell (runGate/inspectPlugin) is exercised
|
|
// against the live tree by the CLI, not here.
|
|
import { test } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import {
|
|
normalizeVersion,
|
|
extractBadgeVersion,
|
|
extractCatalogLabel,
|
|
classifyPlugin,
|
|
extractStatBadges,
|
|
extractCatalogStats,
|
|
} from './check-versions.mjs';
|
|
|
|
test('normalizeVersion strips a leading v', () => {
|
|
assert.equal(normalizeVersion('v5.4.0'), '5.4.0');
|
|
assert.equal(normalizeVersion('5.4.0'), '5.4.0');
|
|
assert.equal(normalizeVersion(' v1.16.0 '), '1.16.0');
|
|
});
|
|
|
|
test('extractBadgeVersion pulls the version from a shields.io badge', () => {
|
|
assert.equal(extractBadgeVersion(''), '5.4.0');
|
|
assert.equal(extractBadgeVersion('no badge here'), null);
|
|
});
|
|
|
|
test('all-consistent plugin → OK', () => {
|
|
const r = classifyPlugin({
|
|
name: 'config-audit', catalogRef: 'v5.4.0', pluginVersion: '5.4.0',
|
|
readmeBadge: '5.4.0', tags: ['v5.4.0', 'v5.3.0'],
|
|
});
|
|
assert.equal(r.status, 'OK');
|
|
assert.ok(!r.findings.some(f => f.level === 'ERROR' || f.level === 'WARN'));
|
|
});
|
|
|
|
test('catalog ref with no matching tag → ERROR (install-breaking)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'voyage', catalogRef: 'v5.5.0', pluginVersion: '5.5.0',
|
|
readmeBadge: '5.5.0', tags: ['v5.1.1'],
|
|
});
|
|
assert.equal(r.status, 'ERROR');
|
|
assert.ok(r.findings.some(f => f.level === 'ERROR' && /no matching git tag/.test(f.msg)));
|
|
});
|
|
|
|
test('plugin.json version != README badge → ERROR (internal corruption)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'x', catalogRef: 'v1.0.0', pluginVersion: '1.0.0',
|
|
readmeBadge: '0.9.0', tags: ['v1.0.0'],
|
|
});
|
|
assert.equal(r.status, 'ERROR');
|
|
assert.ok(r.findings.some(f => f.level === 'ERROR' && /README version-badge/.test(f.msg)));
|
|
});
|
|
|
|
test('catalog ref behind a RELEASED version (tag exists) → WARN, suggests bumping catalog', () => {
|
|
const r = classifyPlugin({
|
|
name: 'x', catalogRef: 'v1.15.0', pluginVersion: '1.16.0',
|
|
readmeBadge: '1.16.0', tags: ['v1.16.0', 'v1.15.0'],
|
|
});
|
|
assert.equal(r.status, 'WARN');
|
|
assert.ok(r.findings.some(f => f.level === 'WARN' && /bump catalog ref/.test(f.msg)));
|
|
});
|
|
|
|
test('plugin.json ahead with no tag of its own → WARN, flags unreleased/untagged', () => {
|
|
const r = classifyPlugin({
|
|
name: 'ms-ai-architect', catalogRef: 'v1.15.0', pluginVersion: '1.16.0',
|
|
readmeBadge: '1.16.0', tags: ['v1.15.0'],
|
|
});
|
|
assert.equal(r.status, 'WARN');
|
|
assert.ok(r.findings.some(f => f.level === 'WARN' && /never tagged|unreleased/.test(f.msg)));
|
|
});
|
|
|
|
test('extractCatalogLabel reads the catalog README label by plugin name', () => {
|
|
const readme = [
|
|
'### [Config-Audit](https://git.fromaitochitta.com/open/config-audit) `v5.7.0`',
|
|
'### [MS AI Architect](https://git.fromaitochitta.com/open/ms-ai-architect) `v1.15.0` `🇳🇴 Norwegian`',
|
|
].join('\n');
|
|
assert.equal(extractCatalogLabel(readme, 'config-audit'), '5.7.0');
|
|
// first vX.Y.Z token wins — trailing flag/lang badge on the same line is ignored
|
|
assert.equal(extractCatalogLabel(readme, 'ms-ai-architect'), '1.15.0');
|
|
// no heading for this plugin → null (check is skipped)
|
|
assert.equal(extractCatalogLabel(readme, 'ghost'), null);
|
|
});
|
|
|
|
test('catalog README label != catalog ref → ERROR (doc misstates installed version)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'config-audit', catalogRef: 'v5.7.0', pluginVersion: '5.7.0',
|
|
readmeBadge: '5.7.0', tags: ['v5.7.0'], catalogLabel: '5.5.0',
|
|
});
|
|
assert.equal(r.status, 'ERROR');
|
|
assert.ok(r.findings.some(f => f.level === 'ERROR' && /README label/.test(f.msg)));
|
|
});
|
|
|
|
test('catalog README label == catalog ref → no label finding (stays OK)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'config-audit', catalogRef: 'v5.7.0', pluginVersion: '5.7.0',
|
|
readmeBadge: '5.7.0', tags: ['v5.7.0'], catalogLabel: '5.7.0',
|
|
});
|
|
assert.equal(r.status, 'OK');
|
|
});
|
|
|
|
test('label check does not fire on the legitimate ref-lags-plugin.json WARN case', () => {
|
|
// ref v1.15.0 lags plugin.json 1.16.0 (WARN), but the label matches the ref → no extra ERROR
|
|
const r = classifyPlugin({
|
|
name: 'ms-ai-architect', catalogRef: 'v1.15.0', pluginVersion: '1.16.0',
|
|
readmeBadge: '1.16.0', tags: ['v1.15.0'], catalogLabel: '1.15.0',
|
|
});
|
|
assert.equal(r.status, 'WARN');
|
|
assert.ok(!r.findings.some(f => f.level === 'ERROR'));
|
|
});
|
|
|
|
test('catalogLabel omitted (legacy callers) → label check skipped', () => {
|
|
const r = classifyPlugin({
|
|
name: 'x', catalogRef: 'v1.0.0', pluginVersion: '1.0.0',
|
|
readmeBadge: '1.0.0', tags: ['v1.0.0'],
|
|
});
|
|
assert.equal(r.status, 'OK');
|
|
});
|
|
|
|
test('plugin repo not found locally → SKIP', () => {
|
|
const r = classifyPlugin({
|
|
name: 'gone', catalogRef: 'v1.0.0', pluginVersion: null,
|
|
readmeBadge: null, tags: null,
|
|
});
|
|
assert.equal(r.status, 'SKIP');
|
|
});
|
|
|
|
test('ERROR dominates WARN when both apply', () => {
|
|
const r = classifyPlugin({
|
|
name: 'x', catalogRef: 'v2.0.0', pluginVersion: '2.1.0',
|
|
readmeBadge: '2.1.0', tags: ['v1.9.0'], // ref v2.0.0 dangling AND catalog behind v2.1.0
|
|
});
|
|
assert.equal(r.status, 'ERROR');
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stat-badge mirroring (added 2026-08-02, after measuring all 11 plugin READMEs
|
|
// + the catalog's 11 stat lines). Rule chosen by the operator: PER-AXIS. An axis
|
|
// is gated only when the plugin carries a matching badge; a badge-less axis is
|
|
// skipped, not flagged. Measured basis: 21 axis-pairs are badge-covered, and 14
|
|
// axes across 8 plugins are badge-less (voyage 4, ms-ai-architect 3,
|
|
// repo-mailbox 2, ai-psychosis 2, linkedin-studio 1, claude-design 1,
|
|
// graceful-handoff 1) — so a hardcoded per-PLUGIN exception list would have had
|
|
// to name 8 of 11 repos.
|
|
|
|
test('extractStatBadges reads numeric shields badges, ignoring non-numeric ones', () => {
|
|
const readme = [
|
|
'',
|
|
'',
|
|
'',
|
|
'',
|
|
'',
|
|
].join('\n');
|
|
const s = extractStatBadges(readme);
|
|
assert.equal(s.get('command'), 20);
|
|
assert.equal(s.get('test'), 2013);
|
|
// version/platform/license carry non-numeric values → never stat axes
|
|
assert.equal(s.has('version'), false);
|
|
assert.equal(s.has('platform'), false);
|
|
assert.equal(s.has('license'), false);
|
|
});
|
|
|
|
test('extractStatBadges handles a zero value and the -- literal-hyphen escape', () => {
|
|
const readme = [
|
|
'',
|
|
'',
|
|
].join('\n');
|
|
const s = extractStatBadges(readme);
|
|
assert.equal(s.get('hook'), 0, '0 is a real count, not a missing badge');
|
|
assert.equal(s.has('state-helper'), false, 'non-numeric value → not a stat axis');
|
|
});
|
|
|
|
test('extractCatalogStats parses the per-plugin stat line into axis counts', () => {
|
|
const cat = [
|
|
'### [Config-Audit](https://git.fromaitochitta.com/open/config-audit) `v5.13.0`',
|
|
'',
|
|
'Some prose with 99 red herrings in it.',
|
|
'',
|
|
'7 agents · 16 scanners · 21 commands · 1410 tests · [Full documentation →](https://x)',
|
|
'',
|
|
'---',
|
|
].join('\n');
|
|
const s = extractCatalogStats(cat, 'config-audit');
|
|
assert.equal(s.get('agent'), 7);
|
|
assert.equal(s.get('scanner'), 16);
|
|
assert.equal(s.get('command'), 21);
|
|
assert.equal(s.get('test'), 1410);
|
|
assert.equal(s.has('red herring'), false, 'prose above the stat line must not leak in');
|
|
});
|
|
|
|
test('extractCatalogStats reads a parenthetical count as its own axis', () => {
|
|
const cat = [
|
|
'### [MS AI Architect](https://git.fromaitochitta.com/open/ms-ai-architect) `v1.17.0`',
|
|
'',
|
|
'12 agents · 29 commands · 5 skills (389 docs) · 2 hooks · [Full documentation →](https://x)',
|
|
].join('\n');
|
|
const s = extractCatalogStats(cat, 'ms-ai-architect');
|
|
assert.equal(s.get('skill'), 5);
|
|
assert.equal(s.get('reference'), 389, 'doc/docs normalizes onto the reference axis');
|
|
assert.equal(s.get('hook'), 2);
|
|
});
|
|
|
|
test('extractCatalogStats returns an empty map when the plugin has no entry', () => {
|
|
assert.equal(extractCatalogStats('### [Other](https://x/open/other) `v1.0.0`', 'absent').size, 0);
|
|
});
|
|
|
|
test('stat mismatch on a badge-covered axis → ERROR (the config-audit 1441/1410 case)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'config-audit', catalogRef: 'v5.13.0', pluginVersion: '5.13.0',
|
|
readmeBadge: '5.13.0', tags: ['v5.13.0'], catalogLabel: '5.13.0',
|
|
statBadges: new Map([['test', 1441], ['agent', 7]]),
|
|
catalogStats: new Map([['test', 1410], ['agent', 7]]),
|
|
});
|
|
assert.equal(r.status, 'ERROR');
|
|
assert.ok(r.findings.some(f => f.level === 'ERROR' && /test/.test(f.msg) && /1441/.test(f.msg) && /1410/.test(f.msg)));
|
|
});
|
|
|
|
test('every stat axis agreeing → stays OK', () => {
|
|
const r = classifyPlugin({
|
|
name: 'okr', catalogRef: 'v1.8.2', pluginVersion: '1.8.2',
|
|
readmeBadge: '1.8.2', tags: ['v1.8.2'], catalogLabel: '1.8.2',
|
|
statBadges: new Map([['agent', 7], ['command', 14], ['hook', 3], ['reference', 17]]),
|
|
catalogStats: new Map([['agent', 7], ['command', 14], ['hook', 3]]),
|
|
});
|
|
assert.equal(r.status, 'OK', 'a badge the catalog simply does not restate is not a finding');
|
|
});
|
|
|
|
test('badge-less catalog axis is SKIPPED, not flagged (the whole voyage case)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'voyage', catalogRef: 'v5.9.1', pluginVersion: '5.9.1',
|
|
readmeBadge: '5.9.1', tags: ['v5.9.1'], catalogLabel: '5.9.1',
|
|
statBadges: new Map(),
|
|
catalogStats: new Map([['agent', 24], ['command', 6], ['hook', 7], ['test', 500]]),
|
|
});
|
|
assert.equal(r.status, 'OK');
|
|
assert.ok(!r.findings.some(f => f.level === 'ERROR'), 'no badge → no claim to mirror → no error');
|
|
});
|
|
|
|
test('a zero badge still gates (0 != 3 is a real mismatch, not a missing badge)', () => {
|
|
const r = classifyPlugin({
|
|
name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
|
tags: ['v1.0.0'], catalogLabel: '1.0.0',
|
|
statBadges: new Map([['hook', 0]]),
|
|
catalogStats: new Map([['hook', 3]]),
|
|
});
|
|
assert.equal(r.status, 'ERROR');
|
|
});
|
|
|
|
test('an approximate catalog count (N+) gates as a LOWER BOUND', () => {
|
|
const under = classifyPlugin({
|
|
name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
|
tags: ['v1.0.0'], catalogLabel: '1.0.0',
|
|
statBadges: new Map([['test', 400]]),
|
|
catalogStats: new Map([['test', { atLeast: 500 }]]),
|
|
});
|
|
assert.equal(under.status, 'ERROR', '"500+ tests" while the badge says 400 overstates');
|
|
|
|
const over = classifyPlugin({
|
|
name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
|
tags: ['v1.0.0'], catalogLabel: '1.0.0',
|
|
statBadges: new Map([['test', 2013]]),
|
|
catalogStats: new Map([['test', { atLeast: 500 }]]),
|
|
});
|
|
assert.equal(over.status, 'OK', '"500+" is satisfied by any badge >= 500');
|
|
});
|
|
|
|
test('stat maps omitted (legacy callers) → mirroring skipped entirely', () => {
|
|
const r = classifyPlugin({
|
|
name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0',
|
|
readmeBadge: '1.0.0', tags: ['v1.0.0'], catalogLabel: '1.0.0',
|
|
});
|
|
assert.equal(r.status, 'OK');
|
|
});
|