feat(check-versions): gate the catalog's stat lines against plugin badges [skip-docs]
[skip-docs]: the user-facing doc here is CLAUDE.md, not README.md. This is a
maintainer-side consistency gate — the catalog landing page should not describe
it — and CLAUDE.md gains a full paragraph covering the rule, the measured
badge-less gap, and the never-hand-edit-a-stat-line instruction. README.md
changes by exactly one number because the gate found it wrong.
The catalog restates each plugin's counts in a per-plugin stat line, and those
numbers rot silently: nothing compared them to anything. Measured across all 11
plugin READMEs today, config-audit's line claimed 1410 tests while the plugin's
own badge said 1441.
Rule is PER-AXIS, not per-plugin. For each number on a catalog stat line, if the
plugin carries a shields badge for that axis, they must agree (ERROR otherwise);
if it carries no such badge, the axis is skipped. This was measured, not assumed:
21 axis-pairs are badge-covered, but 14 axes across 8 of the 11 plugins are
badge-less (voyage 4, ms-ai-architect 3, repo-mailbox 2, ai-psychosis 2,
linkedin-studio 1, claude-design 1, graceful-handoff 1). A per-plugin exception
list — the original sketch, scoped around voyage alone — would have had to name 8
of 11 repos and be hand-edited for every new axis.
Stated plainly rather than hidden: those 14 axes stay ungated. repo-mailbox's two
were 6 and 251 against a true 8 and 374 (fixed in e9054f8).
Parsing is measured against the real corpus: shields' `--` literal-hyphen escape,
`_`/`%20` spaces, zero as a real count, non-numeric badges (version/platform/
license) excluded, parenthetical asides read as their own axis ("5 skills (389
docs)"), doc/docs/references/reference_docs/knowledge_docs folded onto one axis,
and `N+` read as a lower bound.
Only the catalog's config-audit number changed; the plugin badge is the source.
Tests 106 -> 117 (+11). Gate green at 11 OK / 0 WARN / 0 ERROR.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RDSiMcgLMpEETwtkc86Nym
This commit is contained in:
parent
e9054f8452
commit
fc95cbd300
4 changed files with 255 additions and 7 deletions
|
|
@ -8,6 +8,8 @@ import {
|
|||
extractBadgeVersion,
|
||||
extractCatalogLabel,
|
||||
classifyPlugin,
|
||||
extractStatBadges,
|
||||
extractCatalogStats,
|
||||
} from './check-versions.mjs';
|
||||
|
||||
test('normalizeVersion strips a leading v', () => {
|
||||
|
|
@ -128,3 +130,142 @@ test('ERROR dominates WARN when both apply', () => {
|
|||
});
|
||||
assert.equal(r.status, 'ERROR');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stat-badge mirroring (added 2026-08-02, after measuring all 11 plugin READMEs
|
||||
// + the catalog's 11 stat lines). Rule chosen by the operator: PER-AXIS. An axis
|
||||
// is gated only when the plugin carries a matching badge; a badge-less axis is
|
||||
// skipped, not flagged. Measured basis: 21 axis-pairs are badge-covered, and 14
|
||||
// axes across 8 plugins are badge-less (voyage 4, ms-ai-architect 3,
|
||||
// repo-mailbox 2, ai-psychosis 2, linkedin-studio 1, claude-design 1,
|
||||
// graceful-handoff 1) — so a hardcoded per-PLUGIN exception list would have had
|
||||
// to name 8 of 11 repos.
|
||||
|
||||
test('extractStatBadges reads numeric shields badges, ignoring non-numeric ones', () => {
|
||||
const readme = [
|
||||
'',
|
||||
'',
|
||||
'',
|
||||
'',
|
||||
'',
|
||||
].join('\n');
|
||||
const s = extractStatBadges(readme);
|
||||
assert.equal(s.get('command'), 20);
|
||||
assert.equal(s.get('test'), 2013);
|
||||
// version/platform/license carry non-numeric values → never stat axes
|
||||
assert.equal(s.has('version'), false);
|
||||
assert.equal(s.has('platform'), false);
|
||||
assert.equal(s.has('license'), false);
|
||||
});
|
||||
|
||||
test('extractStatBadges handles a zero value and the -- literal-hyphen escape', () => {
|
||||
const readme = [
|
||||
'',
|
||||
'',
|
||||
].join('\n');
|
||||
const s = extractStatBadges(readme);
|
||||
assert.equal(s.get('hook'), 0, '0 is a real count, not a missing badge');
|
||||
assert.equal(s.has('state-helper'), false, 'non-numeric value → not a stat axis');
|
||||
});
|
||||
|
||||
test('extractCatalogStats parses the per-plugin stat line into axis counts', () => {
|
||||
const cat = [
|
||||
'### [Config-Audit](https://git.fromaitochitta.com/open/config-audit) `v5.13.0`',
|
||||
'',
|
||||
'Some prose with 99 red herrings in it.',
|
||||
'',
|
||||
'7 agents · 16 scanners · 21 commands · 1410 tests · [Full documentation →](https://x)',
|
||||
'',
|
||||
'---',
|
||||
].join('\n');
|
||||
const s = extractCatalogStats(cat, 'config-audit');
|
||||
assert.equal(s.get('agent'), 7);
|
||||
assert.equal(s.get('scanner'), 16);
|
||||
assert.equal(s.get('command'), 21);
|
||||
assert.equal(s.get('test'), 1410);
|
||||
assert.equal(s.has('red herring'), false, 'prose above the stat line must not leak in');
|
||||
});
|
||||
|
||||
test('extractCatalogStats reads a parenthetical count as its own axis', () => {
|
||||
const cat = [
|
||||
'### [MS AI Architect](https://git.fromaitochitta.com/open/ms-ai-architect) `v1.17.0`',
|
||||
'',
|
||||
'12 agents · 29 commands · 5 skills (389 docs) · 2 hooks · [Full documentation →](https://x)',
|
||||
].join('\n');
|
||||
const s = extractCatalogStats(cat, 'ms-ai-architect');
|
||||
assert.equal(s.get('skill'), 5);
|
||||
assert.equal(s.get('reference'), 389, 'doc/docs normalizes onto the reference axis');
|
||||
assert.equal(s.get('hook'), 2);
|
||||
});
|
||||
|
||||
test('extractCatalogStats returns an empty map when the plugin has no entry', () => {
|
||||
assert.equal(extractCatalogStats('### [Other](https://x/open/other) `v1.0.0`', 'absent').size, 0);
|
||||
});
|
||||
|
||||
test('stat mismatch on a badge-covered axis → ERROR (the config-audit 1441/1410 case)', () => {
|
||||
const r = classifyPlugin({
|
||||
name: 'config-audit', catalogRef: 'v5.13.0', pluginVersion: '5.13.0',
|
||||
readmeBadge: '5.13.0', tags: ['v5.13.0'], catalogLabel: '5.13.0',
|
||||
statBadges: new Map([['test', 1441], ['agent', 7]]),
|
||||
catalogStats: new Map([['test', 1410], ['agent', 7]]),
|
||||
});
|
||||
assert.equal(r.status, 'ERROR');
|
||||
assert.ok(r.findings.some(f => f.level === 'ERROR' && /test/.test(f.msg) && /1441/.test(f.msg) && /1410/.test(f.msg)));
|
||||
});
|
||||
|
||||
test('every stat axis agreeing → stays OK', () => {
|
||||
const r = classifyPlugin({
|
||||
name: 'okr', catalogRef: 'v1.8.2', pluginVersion: '1.8.2',
|
||||
readmeBadge: '1.8.2', tags: ['v1.8.2'], catalogLabel: '1.8.2',
|
||||
statBadges: new Map([['agent', 7], ['command', 14], ['hook', 3], ['reference', 17]]),
|
||||
catalogStats: new Map([['agent', 7], ['command', 14], ['hook', 3]]),
|
||||
});
|
||||
assert.equal(r.status, 'OK', 'a badge the catalog simply does not restate is not a finding');
|
||||
});
|
||||
|
||||
test('badge-less catalog axis is SKIPPED, not flagged (the whole voyage case)', () => {
|
||||
const r = classifyPlugin({
|
||||
name: 'voyage', catalogRef: 'v5.9.1', pluginVersion: '5.9.1',
|
||||
readmeBadge: '5.9.1', tags: ['v5.9.1'], catalogLabel: '5.9.1',
|
||||
statBadges: new Map(),
|
||||
catalogStats: new Map([['agent', 24], ['command', 6], ['hook', 7], ['test', 500]]),
|
||||
});
|
||||
assert.equal(r.status, 'OK');
|
||||
assert.ok(!r.findings.some(f => f.level === 'ERROR'), 'no badge → no claim to mirror → no error');
|
||||
});
|
||||
|
||||
test('a zero badge still gates (0 != 3 is a real mismatch, not a missing badge)', () => {
|
||||
const r = classifyPlugin({
|
||||
name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
||||
tags: ['v1.0.0'], catalogLabel: '1.0.0',
|
||||
statBadges: new Map([['hook', 0]]),
|
||||
catalogStats: new Map([['hook', 3]]),
|
||||
});
|
||||
assert.equal(r.status, 'ERROR');
|
||||
});
|
||||
|
||||
test('an approximate catalog count (N+) gates as a LOWER BOUND', () => {
|
||||
const under = classifyPlugin({
|
||||
name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
||||
tags: ['v1.0.0'], catalogLabel: '1.0.0',
|
||||
statBadges: new Map([['test', 400]]),
|
||||
catalogStats: new Map([['test', { atLeast: 500 }]]),
|
||||
});
|
||||
assert.equal(under.status, 'ERROR', '"500+ tests" while the badge says 400 overstates');
|
||||
|
||||
const over = classifyPlugin({
|
||||
name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
||||
tags: ['v1.0.0'], catalogLabel: '1.0.0',
|
||||
statBadges: new Map([['test', 2013]]),
|
||||
catalogStats: new Map([['test', { atLeast: 500 }]]),
|
||||
});
|
||||
assert.equal(over.status, 'OK', '"500+" is satisfied by any badge >= 500');
|
||||
});
|
||||
|
||||
test('stat maps omitted (legacy callers) → mirroring skipped entirely', () => {
|
||||
const r = classifyPlugin({
|
||||
name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0',
|
||||
readmeBadge: '1.0.0', tags: ['v1.0.0'], catalogLabel: '1.0.0',
|
||||
});
|
||||
assert.equal(r.status, 'OK');
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue