fix(catalog): release gate can no longer be green without measuring anything
check-versions.mjs reported "12 SKIP, exit 0" on a fresh clone with no sibling plugin repos — a run that verified nothing was indistinguishable from a clean run. Any SKIP now fails the gate by default (--allow-skip opts in explicitly), and the summary always reports "verified N/M". Also adds a homepage-resolves check on plugin.json (dead-link ERROR, verified live against voyage's stale pre-polyrepo URL); the field stays optional since 11 of 12 plugins don't carry it yet. gateOutcome is split out as a pure function so both directions (all-SKIP fails, fully-verified-clean still passes) are unit-tested without needing a real sibling-repo layout on disk.
This commit is contained in:
parent
4509bb6096
commit
0b0bf24137
3 changed files with 210 additions and 13 deletions
|
|
@ -11,6 +11,7 @@ import {
|
|||
extractStatBadges,
|
||||
pickStatSource,
|
||||
extractCatalogStats,
|
||||
gateOutcome,
|
||||
} from './check-versions.mjs';
|
||||
|
||||
test('normalizeVersion strips a leading v', () => {
|
||||
|
|
@ -292,3 +293,109 @@ test('pickStatSource falls back to the working tree only when the ref cannot be
|
|||
test('an empty README at the ref is a real read, not a failed one', () => {
|
||||
assert.equal(pickStatSource({ atRef: '', atWorktree: 'badge/scanners-22-cyan' }), '');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// gateOutcome — the fix for the UNMEASURED-reads-as-green bug found 2026-08-18.
|
||||
// A fresh clone with no sibling repos cloned produced "12 plugins — 0 OK, 0
|
||||
// WARN, 0 ERROR, 12 SKIP, exit 0": a run that verified NOTHING was
|
||||
// indistinguishable from a run that verified everything and found it clean.
|
||||
// Both directions must hold, or a fix that only
|
||||
// reddens the all-SKIP case could just as easily redden (or silently pass) the
|
||||
// all-verified case too.
|
||||
|
||||
test('gateOutcome: all-SKIP (nothing verified) fails even with zero ERROR/WARN', () => {
|
||||
const results = [{ status: 'SKIP' }, { status: 'SKIP' }, { status: 'SKIP' }];
|
||||
const o = gateOutcome(results);
|
||||
assert.equal(o.hasError, false);
|
||||
assert.equal(o.skipped, 3);
|
||||
assert.equal(o.verified, 0);
|
||||
assert.equal(o.total, 3);
|
||||
assert.equal(o.failed, true, 'a run that verified nothing must not report green');
|
||||
});
|
||||
|
||||
test('gateOutcome: fully-verified all-OK run stays green (the fix must not break the normal case)', () => {
|
||||
const results = [{ status: 'OK' }, { status: 'OK' }, { status: 'WARN' }];
|
||||
const o = gateOutcome(results);
|
||||
assert.equal(o.skipped, 0);
|
||||
assert.equal(o.verified, 3);
|
||||
assert.equal(o.failed, false, 'no ERROR, no SKIP, non-strict → still green');
|
||||
});
|
||||
|
||||
test('gateOutcome: ERROR still fails regardless of skip count', () => {
|
||||
const o = gateOutcome([{ status: 'ERROR' }, { status: 'SKIP' }]);
|
||||
assert.equal(o.failed, true);
|
||||
});
|
||||
|
||||
test('gateOutcome: --allow-skip lets a partially-unverified run pass deliberately', () => {
|
||||
const results = [{ status: 'OK' }, { status: 'SKIP' }];
|
||||
const withoutFlag = gateOutcome(results);
|
||||
const withFlag = gateOutcome(results, { allowSkip: true });
|
||||
assert.equal(withoutFlag.failed, true, 'default: unverified plugin blocks green');
|
||||
assert.equal(withFlag.failed, false, 'explicit opt-in: caller accepts partial verification');
|
||||
});
|
||||
|
||||
test('gateOutcome: strict mode still fails on WARN even with everything verified', () => {
|
||||
const o = gateOutcome([{ status: 'WARN' }], { strict: true });
|
||||
assert.equal(o.failed, true);
|
||||
});
|
||||
|
||||
test('gateOutcome: verified/total denominator is reported for the summary line', () => {
|
||||
const o = gateOutcome([{ status: 'OK' }, { status: 'OK' }, { status: 'SKIP' }]);
|
||||
assert.equal(o.total, 3);
|
||||
assert.equal(o.verified, 2);
|
||||
assert.equal(o.skipped, 1);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Manifest `homepage` must resolve when present (order finding 2 — voyage's
|
||||
// manifest pointed at a pre-polyrepo path that 404s). The field stays OPTIONAL
|
||||
// (11 of 12 plugins omit it) — this is a deliberate policy call made in this
|
||||
// session, not an oversight: making it required would flip 11 plugins to ERROR
|
||||
// and, via release-plugin.mjs's ERROR-set pre-flight, block every release in
|
||||
// the marketplace until all 11 sibling repos re-tag. See check-versions.mjs
|
||||
// for the same note at the check site.
|
||||
|
||||
test('homepage present and reachable → no finding (known-positive)', () => {
|
||||
const r = classifyPlugin({
|
||||
name: 'voyage', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
||||
tags: ['v1.0.0'], homepage: 'https://git.fromaitochitta.com/open/voyage', homepageReachable: true,
|
||||
});
|
||||
assert.equal(r.status, 'OK');
|
||||
assert.ok(!r.findings.some(f => /homepage/.test(f.msg)));
|
||||
});
|
||||
|
||||
test('homepage present and dead → ERROR (known-negative, the voyage 404 case)', () => {
|
||||
const r = classifyPlugin({
|
||||
name: 'voyage', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
||||
tags: ['v1.0.0'],
|
||||
homepage: 'https://git.fromaitochitta.com/open/ktg-plugin-marketplace/src/branch/main/plugins/voyage',
|
||||
homepageReachable: false,
|
||||
});
|
||||
assert.equal(r.status, 'ERROR');
|
||||
assert.ok(r.findings.some(f => f.level === 'ERROR' && /homepage/.test(f.msg) && /does not resolve/.test(f.msg)));
|
||||
});
|
||||
|
||||
test('homepage absent → no finding (field is optional, not every plugin must carry it)', () => {
|
||||
const r = classifyPlugin({
|
||||
name: 'config-audit', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
||||
tags: ['v1.0.0'], homepage: null, homepageReachable: null,
|
||||
});
|
||||
assert.equal(r.status, 'OK');
|
||||
assert.ok(!r.findings.some(f => /homepage/.test(f.msg)));
|
||||
});
|
||||
|
||||
test('homepage present but unchecked (network unreachable) → not silently treated as reachable, but does not false-fail', () => {
|
||||
const r = classifyPlugin({
|
||||
name: 'voyage', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0',
|
||||
tags: ['v1.0.0'], homepage: 'https://git.fromaitochitta.com/open/voyage', homepageReachable: null,
|
||||
});
|
||||
assert.equal(r.status, 'OK', 'unmeasured must not be reported as a confirmed dead link');
|
||||
assert.ok(!r.findings.some(f => /homepage/.test(f.msg)));
|
||||
});
|
||||
|
||||
test('homepage omitted entirely (legacy callers) → check skipped, stays OK', () => {
|
||||
const r = classifyPlugin({
|
||||
name: 'x', catalogRef: 'v1.0.0', pluginVersion: '1.0.0', readmeBadge: '1.0.0', tags: ['v1.0.0'],
|
||||
});
|
||||
assert.equal(r.status, 'OK');
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue