ktg-plugin-marketplace/scripts/check-versions.test.mjs

// Tests for the marketplace version-consistency gate.
// Pure classifier is the unit under test — I/O shell (runGate/inspectPlugin) is exercised
// against the live tree by the CLI, not here.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
  normalizeVersion,
  extractBadgeVersion,
  extractCatalogLabel,
  classifyPlugin,
  extractStatBadges,
  pickStatSource,
  extractCatalogStats,
  gateOutcome,
  statCoverage,
  coverageSummary,
} from './check-versions.mjs';

test('normalizeVersion strips a leading v', () => {
  assert.equal(normalizeVersion('v5.4.0'), '5.4.0');
  assert.equal(normalizeVersion('5.4.0'), '5.4.0');
  assert.equal(normalizeVersion('  v1.16.0 '), '1.16.0');
});

test('extractBadgeVersion pulls the version from a shields.io badge', () => {
  assert.equal(extractBadgeVersion('![Version](https://img.shields.io/badge/version-5.4.0-blue)'), '5.4.0');
  assert.equal(extractBadgeVersion('no badge here'), null);
});

test('all-consistent plugin → OK', () => {
  const r = classifyPlugin({
    name: 'config-audit', catalogRef: 'v5.4.0', pluginVersion: '5.4.0',
    readmeBadge: '5.4.0', tags: ['v5.4.0', 'v5.3.0'],
  });
  assert.equal(r.status, 'OK');
  assert.ok(!r.findings.some(f => f.level === 'ERROR' || f.level === 'WARN'));
});

test('catalog ref with no matching tag → ERROR (install-breaking)', () => {
  const r = classifyPlugin({
    name: 'voyage', catalogRef: 'v5.5.0', pluginVersion: '5.5.0',
    readmeBadge: '5.5.0', tags: ['v5.1.1'],
  });
  assert.equal(r.status, 'ERROR');
  assert.ok(r.findings.some(f => f.level === 'ERROR' && /no matching git tag/.test(f.msg)));
});

test('plugin.json version != README badge → ERROR (internal corruption)', () => {
  const r = classifyPlugin({
    name: 'x', catalogRef: 'v1.0.0', pluginVersion: '1.0.0',
    readmeBadge: '0.9.0', tags: ['v1.0.0'],
  });
  assert.equal(r.status, 'ERROR');
  assert.ok(r.findings.some(f => f.level === 'ERROR' && /README version-badge/.test(f.msg)));
});

test('catalog ref behind a RELEASED version (tag exists) → WARN, suggests bumping catalog', () => {
  const r = classifyPlugin({
    name: 'x', catalogRef: 'v1.15.0', pluginVersion: '1.16.0',
    readmeBadge: '1.16.0', tags: ['v1.16.0', 'v1.15.0'],
  });
  assert.equal(r.status, 'WARN');
  assert.ok(r.findings.some(f => f.level === 'WARN' && /bump catalog ref/.test(f.msg)));
});

test('plugin.json ahead with no tag of its own → WARN, flags unreleased/untagged', () => {
  const r = classifyPlugin({
    name: 'ms-ai-architect', catalogRef: 'v1.15.0', pluginVersion: '1.16.0',
    readmeBadge: '1.16.0', tags: ['v1.15.0'],
  });
  assert.equal(r.status, 'WARN');
  assert.ok(r.findings.some(f => f.level === 'WARN' && /never tagged|unreleased/.test(f.msg)));
});

test('extractCatalogLabel reads the catalog README label by plugin name', () => {
  const readme = [
    '### [Config-Audit](https://git.fromaitochitta.com/open/config-audit) `v5.7.0`',
    '### [MS AI Architect](https://git.fromaitochitta.com/open/ms-ai-architect) `v1.15.0` `🇳🇴 Norwegian`',
  ].join('\n');
  assert.equal(extractCatalogLabel(readme, 'config-audit'), '5.7.0');
  // first vX.Y.Z token wins — trailing flag/lang badge on the same line is ignored
  assert.equal(extractCatalogLabel(readme, 'ms-ai-architect'), '1.15.0');
  // no heading for this plugin → null (check is skipped)
  assert.equal(extractCatalogLabel(readme, 'ghost'), null);
});

test('catalog README label != catalog ref → ERROR (doc misstates installed version)', () => {
  const r = classifyPlugin({
    name: 'config-audit', catalogRef: 'v5.7.0', pluginVersion: '5.7.0',
    readmeBadge: '5.7.0', tags: ['v5.7.0'], catalogLabel: '5.5.0',
  });
  assert.equal(r.status, 'ERROR');
  assert.ok(r.findings.some(f => f.level === 'ERROR' && /README label/.test(f.msg)));
});

test('catalog README label == catalog ref → no label finding (stays OK)', () => {
  const r = classifyPlugin({
    name: 'config-audit', catalogRef: 'v5.7.0', pluginVersion: '5.7.0',
    readmeBadge: '5.7.0', tags: ['v5.7.0'], catalogLabel: '5.7.0',
  });
  assert.equal(r.status, 'OK');
});

test('label check does not fire on the legitimate ref-lags-plugin.json WARN case', () => {
  // ref v1.15.0 lags plugin.json 1.16.0 (WARN), but the label matches the ref → no extra ERROR
  const r = classifyPlugin({
    name: 'ms-ai-architect', catalogRef: 'v1.15.0', pluginVersion: '1.16.0',
    readmeBadge: '1.16.0', tags: ['v1.15.0'], catalogLabel: '1.15.0',
  });
  assert.equal(r.status, 'WARN');
  assert.ok(!r.findings.some(f => f.level === 'ERROR'));
});

test('catalogLabel omitted (legacy callers) → label check skipped', () => {
  const r = classifyPlugin({
    name: 'x', catalogRef: 'v1.0.0', pluginVersion: '1.0.0',
    readmeBadge: '1.0.0', tags: ['v1.0.0'],
  });
  assert.equal(r.status, 'OK');
});

test('plugin repo not found locally → SKIP', () => {
  const r = classifyPlugin({
    name: 'gone', catalogRef: 'v1.0.0', pluginVersion: null,
    readmeBadge: null, tags: null,
  });
  assert.equal(r.status, 'SKIP');
});

test('ERROR dominates WARN when both apply', () => {
  const r = classifyPlugin({
    name: 'x', catalogRef: 'v2.0.0', pluginVersion: '2.1.0',
    readmeBadge: '2.1.0', tags: ['v1.9.0'], // ref v2.0.0 dangling AND catalog behind v2.1.0
  });
  assert.equal(r.status, 'ERROR');
});

// ---------------------------------------------------------------------------
// Stat-badge mirroring (added 2026-08-02, after measuring all 11 plugin READMEs
// + the catalog's 11 stat lines). Rule chosen by the operator: PER-AXIS. An axis
// is gated only when the plugin carries a matching badge; a badge-less axis is
// skipped, not flagged. Measured basis: 21 axis-pairs are badge-covered, and 14
// axes across 8 plugins are badge-less (voyage 4, ms-ai-architect 3,
// repo-mailbox 2, ai-psychosis 2, linkedin-studio 1, claude-design 1,
// graceful-handoff 1) — so a hardcoded per-PLUGIN exception list would have had
// to name 8 of 11 repos.

test('extractStatBadges reads numeric shields badges, ignoring non-numeric ones', () => {
  const readme = [
    '![v](https://img.shields.io/badge/version-7.8.3-blue)',
    '![p](https://img.shields.io/badge/platform-Claude_Code_Plugin-purple)',
    '![c](https://img.shields.io/badge/commands-20-orange)',
    '![t](https://img.shields.io/badge/tests-2013-success)',
    '![l](https://img.shields.io/badge/license-MIT-lightgrey)',
  ].join('\n');
  const s = extractStatBadges(readme);
  assert.equal(s.get('command'), 20);
  assert.equal(s.get('test'), 2013);
  // version/platform/license carry non-numeric values → never stat axes
  assert.equal(s.has('version'), false);
  assert.equal(s.has('platform'), false);
  assert.equal(s.has('license'), false);
});

test('extractStatBadges handles a zero value and the -- literal-hyphen escape', () => {
  const readme = [
    '![h](https://img.shields.io/badge/hooks-0-lightgrey)',
    '![s](https://img.shields.io/badge/STATE--helper-deterministic-cyan)',
  ].join('\n');
  const s = extractStatBadges(readme);
  assert.equal(s.get('hook'), 0, '0 is a real count, not a missing badge');
  assert.equal(s.has('state-helper'), false, 'non-numeric value → not a stat axis');
});

test('extractCatalogStats parses the per-plugin stat line into axis counts', () => {
  const cat = [
    '### [Config-Audit](https://git.fromaitochitta.com/open/config-audit) `v5.13.0`',
    '',
    'Some prose with 99 red herrings in it.',
    '',
    '7 agents · 16 scanners · 21 commands · 1410 tests · [Full documentation →](https://x)',
    '',
    '---',
  ].join('\n');
  const s = extractCatalogStats(cat, 'config-audit');
  assert.equal(s.get('agent'), 7);
  assert.equal(s.get('scanner'), 16);
  assert.equal(s.get('command'), 21);
  assert.equal(s.get('test'), 1410);
  assert.equal(s.has('red herring'), false, 'prose above the stat line must not leak in');
});

test('extractCatalogStats reads a parenthetical count as its own axis', () => {
  const cat = [
    '### [MS AI Architect](https://git.fromaitochitta.com/open/ms-ai-architect) `v1.17.0`',
    '',
    '12 agents · 29 commands · 5 skills (389 docs) · 2 hooks · [Full documentation →](https://x)',
  ].join('\n');
  const s = extractCatalogStats(cat, 'ms-ai-architect');
  assert.equal(s.get('skill'), 5);
  assert.equal(s.get('reference'), 389, 'doc/docs normalizes onto the reference axis');
  assert.equal(s.get('hook'), 2);
});

test('extractCatalogStats returns an empty map when the plugin has no entry', () => {
  assert.equal(extractCatalogStats('### [Other](https://x/open/other) `v1.0.0`', 'absent').size, 0);
});

test('stat mismatch on a badge-covered axis → ERROR (the config-audit 1441/1410 case)', () => {
  const r = classifyPlugin({
    name: 'config-audit', catalogRef: 'v5.13.0', pluginVersion: '5.13.0',
    readmeBadge: '5.13.0', tags: ['v5.13.0'], catalogLabel: '5.13.0',
    statBadges: new Map([['test', 1441], ['agent', 7]]),
    catalogStats: new Map([['test', 1410], ['agent', 7]]),
  });
  assert.equal(r.status, 'ERROR');
  assert.ok(r.findings.some(f => f.level === 'ERROR' && /test/.test(f.msg) && /1441/.test(f.msg) && /1410/.test(f.msg)));
});

test('every stat axis agreeing → stays OK', () => {
  const r = classifyPlugin({
    name: 'okr', catalogRef: 'v1.8.2', pluginVersion: '1.8.2',
    readmeBadge: '1.8.2', tags: ['v1.8.2'], catalogLabel: '1.8.2',
    statBadges: new Map([['agent', 7], ['command', 14], ['hook', 3], ['reference', 17]]),
    catalogStats: new Map([['agent', 7], ['command', 14], ['hook', 3]]),
  });
  assert.equal(r.status, 'OK', 'a badge the catalog simply does not restate is not a finding');
});

test('badge-less catalog axis is SKIPPED, not flagged (the whole voyage case)', () => {
  const r = classifyPlugin({
    name: 'voyage', catalogRef: 'v5.9.1', pluginVersion: '5.9.1',
    readmeBadge: '5.9.1', tags: ['v5.9.1'], catalogLabel: '5.9.1',
    statBadges: new Map(),
    catalogStats: new Map([['agent', 24], ['command', 6], ['hook', 7], ['test', 500]]),
  });
  assert.equal(r.status, 'OK');
  assert.ok(!r.findings.some(f => f.level === 'ERROR'), 'no badge → no claim to mirror → no error');
});

test('a zero badge still gates (0 != 3 is a real mismatch, not a missing badge)', () => {
  const r = classifyPlugin({
    name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
    tags: ['v1.0.0'], catalogLabel: '1.0.0',
    statBadges: new Map([['hook', 0]]),
    catalogStats: new Map([['hook', 3]]),
  });
  assert.equal(r.status, 'ERROR');
});

test('an approximate catalog count (N+) gates as a LOWER BOUND', () => {
  const under = classifyPlugin({
    name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
    tags: ['v1.0.0'], catalogLabel: '1.0.0',
    statBadges: new Map([['test', 400]]),
    catalogStats: new Map([['test', { atLeast: 500 }]]),
  });
  assert.equal(under.status, 'ERROR', '"500+ tests" while the badge says 400 overstates');

  const over = classifyPlugin({
    name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
    tags: ['v1.0.0'], catalogLabel: '1.0.0',
    statBadges: new Map([['test', 2013]]),
    catalogStats: new Map([['test', { atLeast: 500 }]]),
  });
  assert.equal(over.status, 'OK', '"500+" is satisfied by any badge >= 500');
});

test('stat maps omitted (legacy callers) → mirroring skipped entirely', () => {
  const r = classifyPlugin({
    name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0',
    readmeBadge: '1.0.0', tags: ['v1.0.0'], catalogLabel: '1.0.0',
  });
  assert.equal(r.status, 'OK');
});

// Regression, found live 2026-08-02: llm-security had committed past its v7.8.3 tag
// (scanners 23->22, tests 2013->2034) WITHOUT bumping the version. The gate read the
// sibling's working tree and reported the catalog stale — but the catalog documents
// what INSTALLS, and `ref: v7.8.3` still installs 23/2013. The catalog was right and
// the gate was wrong. Stat badges must therefore be read at the pinned ref.
test('pickStatSource prefers the README at the pinned ref over the working tree', () => {
  const atRef = 'https://img.shields.io/badge/scanners-23-cyan';
  const atWorktree = 'https://img.shields.io/badge/scanners-22-cyan';
  assert.equal(pickStatSource({ atRef, atWorktree }), atRef);
  assert.equal(extractStatBadges(pickStatSource({ atRef, atWorktree })).get('scanner'), 23);
});

test('pickStatSource falls back to the working tree only when the ref cannot be read', () => {
  const atWorktree = 'https://img.shields.io/badge/scanners-22-cyan';
  assert.equal(pickStatSource({ atRef: null, atWorktree }), atWorktree);
  assert.equal(pickStatSource({ atRef: null, atWorktree: null }), null);
});

test('an empty README at the ref is a real read, not a failed one', () => {
  assert.equal(pickStatSource({ atRef: '', atWorktree: 'badge/scanners-22-cyan' }), '');
});

// ---------------------------------------------------------------------------
// gateOutcome — the fix for the UNMEASURED-reads-as-green bug found 2026-08-18.
// A fresh clone with no sibling repos cloned produced "12 plugins — 0 OK, 0
// WARN, 0 ERROR, 12 SKIP, exit 0": a run that verified NOTHING was
// indistinguishable from a run that verified everything and found it clean.
// Both directions must hold, or a fix that only
// reddens the all-SKIP case could just as easily redden (or silently pass) the
// all-verified case too.

test('gateOutcome: all-SKIP (nothing verified) fails even with zero ERROR/WARN', () => {
  const results = [{ status: 'SKIP' }, { status: 'SKIP' }, { status: 'SKIP' }];
  const o = gateOutcome(results);
  assert.equal(o.hasError, false);
  assert.equal(o.skipped, 3);
  assert.equal(o.verified, 0);
  assert.equal(o.total, 3);
  assert.equal(o.failed, true, 'a run that verified nothing must not report green');
});

test('gateOutcome: fully-verified all-OK run stays green (the fix must not break the normal case)', () => {
  const results = [{ status: 'OK' }, { status: 'OK' }, { status: 'WARN' }];
  const o = gateOutcome(results);
  assert.equal(o.skipped, 0);
  assert.equal(o.verified, 3);
  assert.equal(o.failed, false, 'no ERROR, no SKIP, non-strict → still green');
});

test('gateOutcome: ERROR still fails regardless of skip count', () => {
  const o = gateOutcome([{ status: 'ERROR' }, { status: 'SKIP' }]);
  assert.equal(o.failed, true);
});

test('gateOutcome: --allow-skip lets a partially-unverified run pass deliberately', () => {
  const results = [{ status: 'OK' }, { status: 'SKIP' }];
  const withoutFlag = gateOutcome(results);
  const withFlag = gateOutcome(results, { allowSkip: true });
  assert.equal(withoutFlag.failed, true, 'default: unverified plugin blocks green');
  assert.equal(withFlag.failed, false, 'explicit opt-in: caller accepts partial verification');
});

test('gateOutcome: strict mode still fails on WARN even with everything verified', () => {
  const o = gateOutcome([{ status: 'WARN' }], { strict: true });
  assert.equal(o.failed, true);
});

test('gateOutcome: verified/total denominator is reported for the summary line', () => {
  const o = gateOutcome([{ status: 'OK' }, { status: 'OK' }, { status: 'SKIP' }]);
  assert.equal(o.total, 3);
  assert.equal(o.verified, 2);
  assert.equal(o.skipped, 1);
});

// ---------------------------------------------------------------------------
// Manifest `homepage` must resolve when present (order finding 2 — voyage's
// manifest pointed at a pre-polyrepo path that 404s). The field stays OPTIONAL
// (11 of 12 plugins omit it) — this is a deliberate policy call made in this
// session, not an oversight: making it required would flip 11 plugins to ERROR
// and, via release-plugin.mjs's ERROR-set pre-flight, block every release in
// the marketplace until all 11 sibling repos re-tag. See check-versions.mjs
// for the same note at the check site.

test('homepage present and reachable → no finding (known-positive)', () => {
  const r = classifyPlugin({
    name: 'voyage', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
    tags: ['v1.0.0'], homepage: 'https://git.fromaitochitta.com/open/voyage', homepageReachable: true,
  });
  assert.equal(r.status, 'OK');
  assert.ok(!r.findings.some(f => /homepage/.test(f.msg)));
});

test('homepage present and dead → ERROR (known-negative, the voyage 404 case)', () => {
  const r = classifyPlugin({
    name: 'voyage', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
    tags: ['v1.0.0'],
    homepage: 'https://git.fromaitochitta.com/open/ktg-plugin-marketplace/src/branch/main/plugins/voyage',
    homepageReachable: false,
  });
  assert.equal(r.status, 'ERROR');
  assert.ok(r.findings.some(f => f.level === 'ERROR' && /homepage/.test(f.msg) && /does not resolve/.test(f.msg)));
});

test('homepage absent → no finding (field is optional, not every plugin must carry it)', () => {
  const r = classifyPlugin({
    name: 'config-audit', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
    tags: ['v1.0.0'], homepage: null, homepageReachable: null,
  });
  assert.equal(r.status, 'OK');
  assert.ok(!r.findings.some(f => /homepage/.test(f.msg)));
});

test('homepage present but unchecked (network unreachable) → not silently treated as reachable, but does not false-fail', () => {
  const r = classifyPlugin({
    name: 'voyage', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
    tags: ['v1.0.0'], homepage: 'https://git.fromaitochitta.com/open/voyage', homepageReachable: null,
  });
  assert.equal(r.status, 'OK', 'unmeasured must not be reported as a confirmed dead link');
  assert.ok(!r.findings.some(f => /homepage/.test(f.msg)));
});

test('homepage omitted entirely (legacy callers) → check skipped, stays OK', () => {
  const r = classifyPlugin({
    name: 'x', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0', tags: ['v1.0.0'],
  });
  assert.equal(r.status, 'OK');
});

// --- Ungated stat claims are NAMED, not silent (operator decision 2026-11-04) -------------
// repo-standard's point, accepted: a mirror that skips a badge-less axis in silence says the
// same thing as a mirror that checked it and found it right — "SKIP is never a pass", one
// level up. The ungated axes are now reported as INFO. INFO never changes a plugin's status
// or the gate's exit code: the catalog may legitimately state a number no badge carries.

test('statCoverage: splits the catalog\'s stated axes into badge-gated and ungated', () => {
  const catalogStats = new Map([['agent', 6], ['scanner', 23], ['test', 2013]]);
  const statBadges = new Map([['scanner', 23], ['knowledge doc', 22]]);
  assert.deepEqual(statCoverage(catalogStats, statBadges), { gated: ['scanner'], ungated: ['agent', 'test'] });
});

test('statCoverage: nothing to say when either side is unknown', () => {
  assert.deepEqual(statCoverage(null, new Map()), { gated: [], ungated: [] });
  assert.deepEqual(statCoverage(new Map([['agent', 6]]), null), { gated: [], ungated: [] });
});

test('classifyPlugin: ungated axes are named in an INFO finding, and the plugin stays OK', () => {
  const r = classifyPlugin({
    name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0', tags: ['v1.0.0'],
    catalogLabel: '1.0.0',
    catalogStats: new Map([['agent', 6], ['scanner', 23], ['hook', 9]]),
    statBadges: new Map([['scanner', 23]]),
  });
  assert.equal(r.status, 'OK');
  const info = r.findings.filter(f => f.level === 'INFO');
  assert.equal(info.length, 1);
  assert.equal(info[0].kind, 'coverage');
  assert.deepEqual(info[0].axes, ['agent', 'hook']);
  assert.match(info[0].msg, /agent, hook/);
  assert.match(info[0].msg, /not gated/);
  assert.ok(r.findings.some(f => f.level === 'OK'), 'the consistent-at line is kept alongside the INFO');
});

test('classifyPlugin: no INFO finding when every stated axis is badge-gated', () => {
  const r = classifyPlugin({
    name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0', tags: ['v1.0.0'],
    catalogLabel: '1.0.0',
    catalogStats: new Map([['scanner', 23]]),
    statBadges: new Map([['scanner', 23]]),
  });
  assert.equal(r.findings.filter(f => f.level === 'INFO').length, 0);
});

test('gateOutcome: INFO (ungated) never fails the gate, not even under --strict', () => {
  const r = classifyPlugin({
    name: 'p', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0', tags: ['v1.0.0'],
    catalogLabel: '1.0.0', catalogStats: new Map([['agent', 6]]), statBadges: new Map(),
  });
  assert.equal(gateOutcome([r], { strict: true }).failed, false);
});

test('coverageSummary: counts gated and ungated claims across plugins, and names the plugins', () => {
  const mk = (name, stats, badges) => classifyPlugin({
    name, catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0', tags: ['v1.0.0'],
    catalogLabel: '1.0.0', catalogStats: new Map(stats), statBadges: new Map(badges),
  });
  const results = [
    mk('a', [['agent', 6], ['scanner', 23]], [['scanner', 23]]),
    mk('b', [['test', 5]], [['test', 5]]),
    mk('c', [['hook', 3], ['command', 16]], []),
    { name: 'd', status: 'SKIP', findings: [{ level: 'SKIP', msg: 'x' }] },
  ];
  assert.deepEqual(coverageSummary(results), { gated: 2, ungated: 3, plugins: ['a', 'c'] });
});