feat(engine)!: a skip says which of the two things it is

`notChecked` merged "deliberately not judged" with "could not run". 0.7.0 took
SKIP out of the severity lattice, which fixed "clean repos look skipped"; this
fixes the same defect one level down, where it was saying so more quietly.
Measured across 19 clones, 35 of 39 skips are deliberate — so
`portfolio-optimiser — OK · 11 not checked` announced eleven unread checks when
all eleven were fixture and out-of-repo links.

Every SKIP finding now carries `skip: 'byDesign' | 'notRun'`, set at the
emission site rather than looked up from its code: VERSION-TAG is emitted at
SKIP with no tags and at ERROR with the wrong one, so a code→kind map has to
re-derive a reason the emitter already had. Untagged falls to `notRun`, the
loud side. Results carry `skips: { byDesign, notRun }`; `notChecked` stays a
number and stays the total, because a consumer doing `notChecked > 0` against
an object gets a silent false.

The summary line names only `notRun` — the deliberate ones keep their own
`NOT JUDGED` heading in the detail output. Landing the split in `--json` alone
would have left the symptom exactly where it was.

VERSION-UNAVAILABLE stays `notRun` by decision, not by default: it is arguably
"no subject to judge", the shape checkReadmeLanguage answers with OK, but
re-levelling it moves a repo's status and is a second behaviour change.

Verified: 169 tests. 19 clones swept with both engines offline — 220 judged
findings, every status, notChecked and buckets identical, no finding differing
beyond the new tag. The exhaustiveness test scans the engine SOURCE, not a
finding set: a corpus test only sees the sites it triggers, and the next
emission site added would take the reader's default invisibly.

BREAKING CHANGE: the summary line's coverage qualifier reads `· N not run` and
counts only un-runnable skips, where it read `· N not checked` over all of
them. `--json` gains `skips`; `notChecked` is unchanged.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DHaQTcaD3w2C6PqVsAmrcv
This commit is contained in:
Kjell Tore Guttormsen 2026-08-09 21:52:00 +02:00
commit f4017bcfe1
8 changed files with 346 additions and 38 deletions

View file

@ -145,6 +145,7 @@ export function checkLinks({ files }, register) {
if (entries.length === 0) {
return [{
level: 'SKIP',
skip: 'notRun',
code: 'LINKS-OPEN-REFS-UNAVAILABLE',
msg: 'no files were enumerated — the `open/` reference check did not run',
}];
@ -198,7 +199,7 @@ export function checkLinks({ files }, register) {
export function checkDescription(description, register) {
if (description === null || description === undefined) {
return [{ level: 'SKIP', code: 'DESC-UNAVAILABLE', msg: 'forge description not available — check not run (offline, or the listing failed)' }];
return [{ level: 'SKIP', skip: 'notRun', code: 'DESC-UNAVAILABLE', msg: 'forge description not available — check not run (offline, or the listing failed)' }];
}
const max = register.description_max_codepoints ?? 180;
const n = countCodepoints(description);
@ -278,7 +279,7 @@ export function checkFirstScreen({ readme, name, description, klass }, register)
}
if (description === null || description === undefined) {
findings.push({ level: 'SKIP', code: 'README-DESC', msg: 'forge description not available — opening-line match not checked' });
findings.push({ level: 'SKIP', skip: 'notRun', code: 'README-DESC', msg: 'forge description not available — opening-line match not checked' });
return findings;
}
@ -418,7 +419,7 @@ export function checkInstallBlock({ readme, name, klass }, register) {
export function checkInstallTruth({ name, klass, catalogNames }) {
if (klass !== 'plugin') return [{ level: 'OK', code: 'INSTALL-TRUTH', msg: 'not a marketplace plugin — nothing to resolve' }];
if (!catalogNames) {
return [{ level: 'SKIP', code: 'INSTALL-TRUTH', msg: 'catalog not reachable — cannot verify the install command actually resolves' }];
return [{ level: 'SKIP', skip: 'notRun', code: 'INSTALL-TRUTH', msg: 'catalog not reachable — cannot verify the install command actually resolves' }];
}
if (!catalogNames.includes(name)) {
return [{
@ -516,7 +517,13 @@ export function checkVersionConsistency({ pluginVersion, readmeBadge, changelogT
const findings = [];
const v = pluginVersion ? String(pluginVersion).replace(/^v/, '') : null;
if (!v) {
return [{ level: 'SKIP', code: 'VERSION-UNAVAILABLE', msg: 'no package version found — nothing to compare against' }];
// `notRun` by operator decision 2026-08-09, not by default. "No package
// version to compare against" is arguably a third thing — no SUBJECT to
// judge, the shape checkReadmeLanguage answers with OK — but re-levelling
// it moves a repo's status, and that is a second behaviour change riding
// on this one. Recorded here so the next reader knows it was decided, not
// overlooked.
return [{ level: 'SKIP', skip: 'notRun', code: 'VERSION-UNAVAILABLE', msg: 'no package version found — nothing to compare against' }];
}
if (readmeBadge !== null && readmeBadge !== undefined && readmeBadge !== v) {
@ -529,7 +536,7 @@ export function checkVersionConsistency({ pluginVersion, readmeBadge, changelogT
// Nothing released yet is a state, not a defect — and it must say so rather
// than pass quietly, because "SKIP is never a pass" is the whole discipline.
if (!tags || tags.length === 0) {
findings.push({ level: 'SKIP', code: 'VERSION-TAG', msg: `repo has no tags — cannot verify that v${v} was ever released` });
findings.push({ level: 'SKIP', skip: 'notRun', code: 'VERSION-TAG', msg: `repo has no tags — cannot verify that v${v} was ever released` });
} else if (!tags.includes(`v${v}`)) {
findings.push({ level: 'ERROR', code: 'VERSION-TAG', bucket: 'broken', msg: `no tag \`v${v}\` — the documented version was never released (tags: ${tags.slice(-3).join(', ')})` });
}
@ -685,6 +692,7 @@ export function checkReadmeLanguage({ readme, name }, register) {
if (hits[declared] < hits[other] * LANG_MIN_RATIO) {
return [{
level: 'SKIP',
skip: 'notRun',
code: 'README-LANGUAGE-UNDECIDABLE',
msg: `README mixes languages too evenly to call (${hits.nb} nb vs ${hits.en} en) — declared \`${declared}\`, unverified`,
}];
@ -867,6 +875,7 @@ export function checkInternalLinks({ files, present }) {
if (resolved === null) {
findings.push({
level: 'SKIP',
skip: 'byDesign',
code: 'LINK-OUTSIDE-REPO',
msg: `${path}:${i + 1}\`${clean}\` points outside this repo; the gate sees one repo and cannot resolve it`,
});
@ -876,6 +885,7 @@ export function checkInternalLinks({ files, present }) {
if (isFixturePath(path)) {
findings.push({
level: 'SKIP',
skip: 'byDesign',
code: 'LINK-INTERNAL-FIXTURE',
msg: `${path}:${i + 1} — link points at \`${clean}\` (${resolved}), which is not a tracked file; ${path} is a test/fixture path, so this is presumed intentional and not judged`,
});
@ -920,6 +930,36 @@ export function notCheckedOf(findings) {
return (findings ?? []).filter((f) => f.level === 'SKIP').length;
}
// The coverage axis is really two facts, and merging them made a repo look
// unread when nothing was: `portfolio-optimiser — OK · 11 not checked`, all
// eleven of them links the gate declines to judge on purpose. org-ops named
// the split (20260809T124015Z, observation 2) without a name for it.
//
// byDesign the check saw the thing and declined — it can never become a
// verdict and nobody has an action. Out-of-repo links, fixture
// paths.
// notRun a re-run or an operator action turns it into a verdict. An
// unreachable catalog, an unregistered repo, a repo with no tags.
//
// The kind is read off the finding, never off its code: `VERSION-TAG` is
// emitted at SKIP with no tags and at ERROR with the wrong one, so a
// code→kind map would have to re-derive a reason the emission site already
// had. Untagged falls to `notRun` — the loud side, because a skip of unknown
// kind must not inherit "deliberate, nothing to see".
export function groupSkips(findings) {
const out = { byDesign: [], notRun: [] };
for (const f of findings ?? []) {
if (f.level !== 'SKIP') continue;
out[f.skip === 'byDesign' ? 'byDesign' : 'notRun'].push(f);
}
return out;
}
export function skipsOf(findings) {
const g = groupSkips(findings);
return { byDesign: g.byDesign.length, notRun: g.notRun.length };
}
export function bucketsOf(findings) {
const out = { broken: 0, missing: 0, weakening: 0 };
for (const f of findings ?? []) {
@ -940,9 +980,11 @@ export function classifyRepo(
traits: [],
status: 'SKIP',
notChecked: 1,
skips: { byDesign: 0, notRun: 1 },
buckets: { broken: 0, missing: 0, weakening: 0 },
findings: [{
level: 'SKIP',
skip: 'notRun',
code: 'REPO-UNREGISTERED',
msg: `\`${name}\` is not in the register — class unknown, so no class-specific rule can be applied. Add it to register/repos.json (or run --refresh).`,
}],
@ -972,7 +1014,11 @@ export function classifyRepo(
klass,
traits,
status: levelOf(findings),
// Still a NUMBER, and still the total. A consumer doing `notChecked > 0`
// against an object gets a silent false — the same class of quiet wrong
// answer this whole axis exists to remove.
notChecked: notCheckedOf(findings),
skips: skipsOf(findings),
buckets: bucketsOf(findings),
findings,
};
@ -1186,13 +1232,24 @@ const MARK = { OK: '✓', WARN: '!', ERROR: '✗', SKIP: '·' };
// Both axes on the one line a sweep actually reads. Letting `status` mean
// judgement fixed "clean repos look skipped"; printing a bare OK next to a
// check that never ran would trade it for "skipped checks look clean", which is
// the worse direction. Absent `notChecked` is neither zero nor a crash — a
// result from before this axis existed prints the old line, not "undefined".
// the worse direction.
//
// Since 0.8.0 the line names only what someone has an ACTION on (operator
// decision 2026-08-09). A deliberate skip is a recorded decision, not an
// unread check, and eleven of them behind an otherwise clean repo said the
// opposite on every row of the sweep. They are not silenced: they keep their
// own sub-heading in the body, which is where "exposure, not silence" lives.
//
// Two generations of older result objects still print correctly, and neither
// absence reads as zero: no `skips` falls back to the 0.7.0 total, no
// `notChecked` to the line from before coverage existed at all.
export function headerLine(result, engineVersion, engineCommit = null) {
const klass = result.klass ? ` [${result.klass}]` : '';
const traits = result.traits?.length ? ` {${result.traits.join(', ')}}` : '';
const sha = engineCommit ? ` @${String(engineCommit).slice(0, 7)}` : '';
const coverage = result.notChecked > 0 ? ` · ${result.notChecked} not checked` : '';
const coverage = result.skips
? (result.skips.notRun > 0 ? ` · ${result.skips.notRun} not run` : '')
: (result.notChecked > 0 ? ` · ${result.notChecked} not checked` : '');
return `${MARK[result.status]} ${result.name}${klass}${traits}${result.status}${coverage} (repo-standard v${engineVersion}${sha})`;
}
@ -1214,10 +1271,17 @@ function render(result, engineVersion, engineCommit) {
for (const f of inBucket) console.log(` ${mark[f.level]} ${f.level} ${f.code}: ${f.msg}`);
}
const skipped = result.findings.filter((f) => f.level === 'SKIP');
if (skipped.length) {
// Two sub-headings, because the header line no longer carries the deliberate
// ones. This is the only place they are visible, and a decision nobody can
// see reads exactly like a check that silently stopped running.
const skipped = groupSkips(result.findings);
if (skipped.notRun.length) {
console.log('\n NOT CHECKED — these are not passes');
for (const f of skipped) console.log(` ${mark.SKIP} ${f.code}: ${f.msg}`);
for (const f of skipped.notRun) console.log(` ${mark.SKIP} ${f.code}: ${f.msg}`);
}
if (skipped.byDesign.length) {
console.log('\n NOT JUDGED — deliberately outside what this gate decides');
for (const f of skipped.byDesign) console.log(` ${mark.SKIP} ${f.code}: ${f.msg}`);
}
const okCount = result.findings.filter((f) => f.level === 'OK').length;

View file

@ -7,6 +7,7 @@
// They are the reason this gate has three outcomes instead of a boolean.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import {
countCodepoints,
normalizeRepoRef,
@ -35,6 +36,8 @@ import {
withEngineVersion,
readEngineCommit,
loadRegister,
groupSkips,
skipsOf,
} from './repo-standard-check.mjs';
const REGISTER = {
@ -1557,3 +1560,146 @@ test('readEngineCommit returns null outside a git checkout instead of throwing',
// a tarball or a vendored copy has no HEAD, and that is not a crash.
assert.equal(readEngineCommit('/nonexistent-path-for-repo-standard-test'), null);
});
// ------------------------------------------------- the two kinds of SKIP
// v0.7.0 took SKIP out of the severity lattice and gave coverage its own axis,
// which fixed "clean repos look skipped". It left one thing merged that org-ops
// had already named (20260809T124015Z, observation 2): a SKIP that means
// "deliberately not judged" is not the same fact as a SKIP that means "could
// not run". Measured over the 05 census, 35 of 39 skips are the first kind —
// so `portfolio-optimiser — OK · 11 not checked` reads as eleven unread checks
// when all eleven are links the gate declines to judge on purpose.
//
// The tag is set at the EMISSION site, never looked up from the code. Codes are
// not the carrier of that meaning: `VERSION-TAG` is emitted at SKIP when there
// are no tags and at ERROR when the tag is missing, so a code→kind map cannot
// express the split without re-deriving the reason it already knew.
test('groupSkips separates the deliberate from the un-runnable', () => {
const g = groupSkips([
{ level: 'SKIP', skip: 'byDesign', code: 'LINK-OUTSIDE-REPO' },
{ level: 'SKIP', skip: 'notRun', code: 'INSTALL-TRUTH' },
{ level: 'SKIP', skip: 'byDesign', code: 'LINK-INTERNAL-FIXTURE' },
]);
assert.deepEqual(g.byDesign.map((f) => f.code), ['LINK-OUTSIDE-REPO', 'LINK-INTERNAL-FIXTURE']);
assert.deepEqual(g.notRun.map((f) => f.code), ['INSTALL-TRUTH']);
});
test('groupSkips ignores judged findings — this axis exists only under SKIP', () => {
const g = groupSkips([
{ level: 'ERROR', code: 'VERSION-TAG', bucket: 'broken' },
{ level: 'OK', code: 'VERSION' },
]);
assert.deepEqual(g, { byDesign: [], notRun: [] });
});
// An untagged SKIP can only come from an older engine or a site the source scan
// below has not caught yet. It lands in `notRun`, which is the loud side: a
// skip of unknown kind must not inherit "deliberate, nothing to see".
test('an untagged SKIP counts as un-runnable, not as deliberate', () => {
assert.deepEqual(skipsOf([{ level: 'SKIP', code: 'MYSTERY' }]), { byDesign: 0, notRun: 1 });
});
test('skipsOf counts both kinds beside the status, same as bucketsOf does', () => {
const counts = skipsOf([
{ level: 'SKIP', skip: 'byDesign', code: 'LINK-OUTSIDE-REPO' },
{ level: 'SKIP', skip: 'byDesign', code: 'LINK-INTERNAL-FIXTURE' },
{ level: 'SKIP', skip: 'notRun', code: 'VERSION-TAG' },
{ level: 'OK', code: 'VERSION' },
]);
assert.deepEqual(counts, { byDesign: 2, notRun: 1 });
});
// The source scan is the test that survives the next quarter. A corpus test
// only sees the sites it manages to trigger; the eleventh emission site added
// later is invisible to it and would silently take the reader's default.
test('every SKIP the engine can emit is tagged where it is emitted', () => {
const src = readFileSync(new URL('./repo-standard-check.mjs', import.meta.url), 'utf8');
const untagged = [];
const re = /level: 'SKIP'/g;
for (const m of src.matchAll(re)) {
// Convention, so this stays checkable: the tag sits between `level` and
// `code` in the object literal. Anything else reads as untagged.
const tail = src.slice(m.index, src.indexOf('code:', m.index));
if (!/skip: '(byDesign|notRun)'/.test(tail)) {
untagged.push(src.slice(0, m.index).split('\n').length);
}
}
assert.deepEqual(untagged, [], `untagged SKIP emission at line(s) ${untagged.join(', ')}`);
});
// Which side each site lands on. The discriminator is not "how bad" but "can
// this ever become a verdict, and does anyone have an action?".
test('a link out of the repo is deliberate — the gate sees one repo by design', () => {
const f = checkInternalLinks({ files: { 'README.md': '[x](../sibling/file.md)' }, present: ['README.md'] });
const hit = f.find((x) => x.code === 'LINK-OUTSIDE-REPO');
assert.equal(hit.skip, 'byDesign');
});
test('a fixture-path dead link is deliberate — presumed intentional, nothing to fix', () => {
const f = checkInternalLinks({ files: { 'tests/plan.md': '[x](gone.md)' }, present: ['tests/plan.md'] });
assert.equal(f.find((x) => x.code === 'LINK-INTERNAL-FIXTURE').skip, 'byDesign');
});
test('no tags yet is un-runnable — a release resolves it', () => {
const f = checkVersionConsistency({ pluginVersion: '0.7.1', readmeBadge: null, changelogTop: null, tags: [] });
assert.equal(f.find((x) => x.code === 'VERSION-TAG').skip, 'notRun');
});
// Kept in `notRun` deliberately (operator decision 2026-08-09). It is arguably
// a third thing — "no subject to judge", the shape `checkReadmeLanguage` calls
// OK rather than SKIP — but re-levelling it moves `status` for a repo and is a
// second behaviour change; it does not ride on this one.
test('no package version is un-runnable, not deliberate', () => {
const f = checkVersionConsistency({ pluginVersion: null, readmeBadge: null, changelogTop: null, tags: [] });
assert.equal(f.find((x) => x.code === 'VERSION-UNAVAILABLE').skip, 'notRun');
});
test('an unreachable catalog is un-runnable — a second run resolves it', () => {
const f = checkInstallTruth({ name: 'repo-mailbox', klass: 'plugin', catalogNames: null });
assert.equal(f.find((x) => x.code === 'INSTALL-TRUTH').skip, 'notRun');
});
test('an unregistered repo is un-runnable — registering it resolves it', () => {
const r = classifyRepo({ name: 'stranger', files: {}, present: [], description: null }, REGISTER);
assert.equal(r.findings[0].skip, 'notRun');
});
test('classifyRepo carries both skip counts, and notChecked stays the unchanged total', () => {
// `notChecked` must keep being a NUMBER. A consumer doing `notChecked > 0` on
// an object gets a silent false, which would kill the header qualifier the
// same way the old status hid clean repos.
const r = classifyRepo({ name: 'stranger', files: {}, present: [], description: null }, REGISTER);
assert.equal(typeof r.notChecked, 'number');
assert.deepEqual(r.skips, { byDesign: 0, notRun: 1 });
assert.equal(r.notChecked, r.skips.byDesign + r.skips.notRun);
});
// The header line is the whole defect. If the split lands only in `--json`, the
// symptom org-ops actually reads survives untouched. Operator decision
// 2026-08-09: the line names only what someone has an action on. The deliberate
// ones are not silenced — they keep their own sub-heading in the body, where
// "exposure, not silence" actually lives.
test('headerLine names the checks that could not run', () => {
const line = headerLine(
{ name: 'x', klass: 'plugin', traits: [], status: 'ERROR', notChecked: 11, skips: { byDesign: 9, notRun: 2 } },
'0.8.0',
);
assert.match(line, /2 not run/);
});
test('headerLine says nothing when every skip was a deliberate one', () => {
const line = headerLine(
{ name: 'portfolio-optimiser', klass: 'tool', traits: [], status: 'OK', notChecked: 11, skips: { byDesign: 11, notRun: 0 } },
'0.8.0',
);
assert.doesNotMatch(line, /not run|not checked|11/);
assert.match(line, /OK/);
});
test('headerLine still prints the pre-0.8.0 line for a result with no skip split', () => {
// Absent `skips` means an older result object, exactly as an absent
// `notChecked` means one older still. Neither is zero.
const line = headerLine({ name: 'okr', klass: 'plugin', traits: [], status: 'OK', notChecked: 1 }, '0.8.0');
assert.match(line, /1 not checked/);
});