open/repo-standard
1
0
Fork 0
Code Issues Releases 7 Activity

feat(engine)!: a skip says which of the two things it is

Browse source 
`notChecked` merged "deliberately not judged" with "could not run". 0.7.0 took
SKIP out of the severity lattice, which fixed "clean repos look skipped"; this
fixes the same defect one level down, where it was saying so more quietly.
Measured across 19 clones, 35 of 39 skips are deliberate — so
`portfolio-optimiser — OK · 11 not checked` announced eleven unread checks when
all eleven were fixture and out-of-repo links.

Every SKIP finding now carries `skip: 'byDesign' | 'notRun'`, set at the
emission site rather than looked up from its code: VERSION-TAG is emitted at
SKIP with no tags and at ERROR with the wrong one, so a code→kind map has to
re-derive a reason the emitter already had. Untagged falls to `notRun`, the
loud side. Results carry `skips: { byDesign, notRun }`; `notChecked` stays a
number and stays the total, because a consumer doing `notChecked > 0` against
an object gets a silent false.

The summary line names only `notRun` — the deliberate ones keep their own
`NOT JUDGED` heading in the detail output. Landing the split in `--json` alone
would have left the symptom exactly where it was.

VERSION-UNAVAILABLE stays `notRun` by decision, not by default: it is arguably
"no subject to judge", the shape checkReadmeLanguage answers with OK, but
re-levelling it moves a repo's status and is a second behaviour change.

Verified: 169 tests. 19 clones swept with both engines offline — 220 judged
findings, every status, notChecked and buckets identical, no finding differing
beyond the new tag. The exhaustiveness test scans the engine SOURCE, not a
finding set: a corpus test only sees the sites it triggers, and the next
emission site added would take the reader's default invisibly.

BREAKING CHANGE: the summary line's coverage qualifier reads `· N not run` and
counts only un-runnable skips, where it read `· N not checked` over all of
them. `--json` gains `skips`; `notChecked` is unchanged.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DHaQTcaD3w2C6PqVsAmrcv
This commit is contained in:
Kjell Tore Guttormsen 2026-08-09 21:52:00 +02:00
commit f4017bcfe1
8 changed files with 346 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

86
scripts/repo-standard-check.mjs
View file

@ -145,6 +145,7 @@ export function checkLinks({ files }, register) {
if (entries.length === 0) {
return [{
level: 'SKIP',
skip: 'notRun',
code: 'LINKS-OPEN-REFS-UNAVAILABLE',
msg: 'no files were enumerated — the `open/` reference check did not run',
}];
@ -198,7 +199,7 @@ export function checkLinks({ files }, register) {
export function checkDescription(description, register) {
if (description === null || description === undefined) {
return [{ level: 'SKIP', code: 'DESC-UNAVAILABLE', msg: 'forge description not available — check not run (offline, or the listing failed)' }];
return [{ level: 'SKIP', skip: 'notRun', code: 'DESC-UNAVAILABLE', msg: 'forge description not available — check not run (offline, or the listing failed)' }];
}
const max = register.description_max_codepoints ?? 180;
const n = countCodepoints(description);
@ -278,7 +279,7 @@ export function checkFirstScreen({ readme, name, description, klass }, register)
}
if (description === null || description === undefined) {
findings.push({ level: 'SKIP', code: 'README-DESC', msg: 'forge description not available — opening-line match not checked' });
findings.push({ level: 'SKIP', skip: 'notRun', code: 'README-DESC', msg: 'forge description not available — opening-line match not checked' });
return findings;
}
@ -418,7 +419,7 @@ export function checkInstallBlock({ readme, name, klass }, register) {
export function checkInstallTruth({ name, klass, catalogNames }) {
if (klass !== 'plugin') return [{ level: 'OK', code: 'INSTALL-TRUTH', msg: 'not a marketplace plugin — nothing to resolve' }];
if (!catalogNames) {
return [{ level: 'SKIP', code: 'INSTALL-TRUTH', msg: 'catalog not reachable — cannot verify the install command actually resolves' }];
return [{ level: 'SKIP', skip: 'notRun', code: 'INSTALL-TRUTH', msg: 'catalog not reachable — cannot verify the install command actually resolves' }];
}
if (!catalogNames.includes(name)) {
return [{
@ -516,7 +517,13 @@ export function checkVersionConsistency({ pluginVersion, readmeBadge, changelogT
const findings = [];
const v = pluginVersion ? String(pluginVersion).replace(/^v/, '') : null;
if (!v) {
return [{ level: 'SKIP', code: 'VERSION-UNAVAILABLE', msg: 'no package version found — nothing to compare against' }];
// `notRun` by operator decision 2026-08-09, not by default. "No package
// version to compare against" is arguably a third thing — no SUBJECT to
// judge, the shape checkReadmeLanguage answers with OK — but re-levelling
// it moves a repo's status, and that is a second behaviour change riding
// on this one. Recorded here so the next reader knows it was decided, not
// overlooked.
return [{ level: 'SKIP', skip: 'notRun', code: 'VERSION-UNAVAILABLE', msg: 'no package version found — nothing to compare against' }];
}
if (readmeBadge !== null && readmeBadge !== undefined && readmeBadge !== v) {
@ -529,7 +536,7 @@ export function checkVersionConsistency({ pluginVersion, readmeBadge, changelogT
// Nothing released yet is a state, not a defect — and it must say so rather
// than pass quietly, because "SKIP is never a pass" is the whole discipline.
if (!tags || tags.length === 0) {
findings.push({ level: 'SKIP', code: 'VERSION-TAG', msg: `repo has no tags — cannot verify that v${v} was ever released` });
findings.push({ level: 'SKIP', skip: 'notRun', code: 'VERSION-TAG', msg: `repo has no tags — cannot verify that v${v} was ever released` });
} else if (!tags.includes(`v${v}`)) {
findings.push({ level: 'ERROR', code: 'VERSION-TAG', bucket: 'broken', msg: `no tag \`v${v}\` — the documented version was never released (tags: ${tags.slice(-3).join(', ')})` });
}
@ -685,6 +692,7 @@ export function checkReadmeLanguage({ readme, name }, register) {
if (hits[declared] < hits[other] * LANG_MIN_RATIO) {
return [{
level: 'SKIP',
skip: 'notRun',
code: 'README-LANGUAGE-UNDECIDABLE',
msg: `README mixes languages too evenly to call (${hits.nb} nb vs ${hits.en} en) — declared \`${declared}\`, unverified`,
}];
@ -867,6 +875,7 @@ export function checkInternalLinks({ files, present }) {
if (resolved === null) {
findings.push({
level: 'SKIP',
skip: 'byDesign',
code: 'LINK-OUTSIDE-REPO',
msg: `${path}:${i + 1} — \`${clean}\` points outside this repo; the gate sees one repo and cannot resolve it`,
});
@ -876,6 +885,7 @@ export function checkInternalLinks({ files, present }) {
if (isFixturePath(path)) {
findings.push({
level: 'SKIP',
skip: 'byDesign',
code: 'LINK-INTERNAL-FIXTURE',
msg: `${path}:${i + 1} — link points at \`${clean}\` (${resolved}), which is not a tracked file; ${path} is a test/fixture path, so this is presumed intentional and not judged`,
});
@ -920,6 +930,36 @@ export function notCheckedOf(findings) {
return (findings ?? []).filter((f) => f.level === 'SKIP').length;
}
// The coverage axis is really two facts, and merging them made a repo look
// unread when nothing was: `portfolio-optimiser — OK · 11 not checked`, all
// eleven of them links the gate declines to judge on purpose. org-ops named
// the split (20260809T124015Z, observation 2) without a name for it.
//
// byDesign the check saw the thing and declined — it can never become a
// verdict and nobody has an action. Out-of-repo links, fixture
// paths.
// notRun a re-run or an operator action turns it into a verdict. An
// unreachable catalog, an unregistered repo, a repo with no tags.
//
// The kind is read off the finding, never off its code: `VERSION-TAG` is
// emitted at SKIP with no tags and at ERROR with the wrong one, so a
// code→kind map would have to re-derive a reason the emission site already
// had. Untagged falls to `notRun` — the loud side, because a skip of unknown
// kind must not inherit "deliberate, nothing to see".
export function groupSkips(findings) {
const out = { byDesign: [], notRun: [] };
for (const f of findings ?? []) {
if (f.level !== 'SKIP') continue;
out[f.skip === 'byDesign' ? 'byDesign' : 'notRun'].push(f);
}
return out;
}
export function skipsOf(findings) {
const g = groupSkips(findings);
return { byDesign: g.byDesign.length, notRun: g.notRun.length };
}
export function bucketsOf(findings) {
const out = { broken: 0, missing: 0, weakening: 0 };
for (const f of findings ?? []) {
@ -940,9 +980,11 @@ export function classifyRepo(
traits: [],
status: 'SKIP',
notChecked: 1,
skips: { byDesign: 0, notRun: 1 },
buckets: { broken: 0, missing: 0, weakening: 0 },
findings: [{
level: 'SKIP',
skip: 'notRun',
code: 'REPO-UNREGISTERED',
msg: `\`${name}\` is not in the register — class unknown, so no class-specific rule can be applied. Add it to register/repos.json (or run --refresh).`,
}],
@ -972,7 +1014,11 @@ export function classifyRepo(
klass,
traits,
status: levelOf(findings),
// Still a NUMBER, and still the total. A consumer doing `notChecked > 0`
// against an object gets a silent false — the same class of quiet wrong
// answer this whole axis exists to remove.
notChecked: notCheckedOf(findings),
skips: skipsOf(findings),
buckets: bucketsOf(findings),
findings,
};
@ -1186,13 +1232,24 @@ const MARK = { OK: '✓', WARN: '!', ERROR: '✗', SKIP: '·' };
// Both axes on the one line a sweep actually reads. Letting `status` mean
// judgement fixed "clean repos look skipped"; printing a bare OK next to a
// check that never ran would trade it for "skipped checks look clean", which is
// the worse direction. Absent `notChecked` is neither zero nor a crash — a
// result from before this axis existed prints the old line, not "undefined".
// the worse direction.
//
// Since 0.8.0 the line names only what someone has an ACTION on (operator
// decision 2026-08-09). A deliberate skip is a recorded decision, not an
// unread check, and eleven of them behind an otherwise clean repo said the
// opposite on every row of the sweep. They are not silenced: they keep their
// own sub-heading in the body, which is where "exposure, not silence" lives.
//
// Two generations of older result objects still print correctly, and neither
// absence reads as zero: no `skips` falls back to the 0.7.0 total, no
// `notChecked` to the line from before coverage existed at all.
export function headerLine(result, engineVersion, engineCommit = null) {
const klass = result.klass ? ` [${result.klass}]` : '';
const traits = result.traits?.length ? ` {${result.traits.join(', ')}}` : '';
const sha = engineCommit ? ` @${String(engineCommit).slice(0, 7)}` : '';
const coverage = result.notChecked > 0 ? ` · ${result.notChecked} not checked` : '';
const coverage = result.skips
? (result.skips.notRun > 0 ? ` · ${result.skips.notRun} not run` : '')
: (result.notChecked > 0 ? ` · ${result.notChecked} not checked` : '');
return `${MARK[result.status]} ${result.name}${klass}${traits} — ${result.status}${coverage} (repo-standard v${engineVersion}${sha})`;
}
@ -1214,10 +1271,17 @@ function render(result, engineVersion, engineCommit) {
for (const f of inBucket) console.log(` ${mark[f.level]} ${f.level} ${f.code}: ${f.msg}`);
}
const skipped = result.findings.filter((f) => f.level === 'SKIP');
if (skipped.length) {
// Two sub-headings, because the header line no longer carries the deliberate
// ones. This is the only place they are visible, and a decision nobody can
// see reads exactly like a check that silently stopped running.
const skipped = groupSkips(result.findings);
if (skipped.notRun.length) {
console.log('\n NOT CHECKED — these are not passes');
for (const f of skipped) console.log(` ${mark.SKIP} ${f.code}: ${f.msg}`);
for (const f of skipped.notRun) console.log(` ${mark.SKIP} ${f.code}: ${f.msg}`);
}
if (skipped.byDesign.length) {
console.log('\n NOT JUDGED — deliberately outside what this gate decides');
for (const f of skipped.byDesign) console.log(` ${mark.SKIP} ${f.code}: ${f.msg}`);
}
const okCount = result.findings.filter((f) => f.level === 'OK').length;