open/repo-standard
1
0
Fork 0
Code Issues Releases 2 Activity

fix(engine): close the linked-badge blind spot, de-tautologize 2 tests

Browse source 
Two remaining minor findings from STATE.md:

checkBadges treated being wrapped in a markdown link as proof of a
real run and skipped BADGE-STATIC-CLAIM entirely — but nothing ever
checked the link actually resolved. The link-target capture group was
missing from the regex outright. Now a linked run-claim badge with a
relative target is resolved against `present` (new BADGE-DEAD-LINK,
ERROR/broken): a dead link is worse than a static badge because it
looks verified. External targets (the ordinary case — a CI provider)
still need the network and stay out of scope, same precedent as
checkInternalLinks.

The "four pre-existing tautological tests" note undercounted on
re-measurement: only two exist ('no class requires a ROADMAP', 'CONTRIBUTING
and CODE_OF_CONDUCT are required by no class'). Both read
`required_files` off the test file's OWN local REGISTER fixture, so
they could only ever check the fixture against itself — a typo in the
real register/repos.json would drift past them silently. Switched
both to `loadRegister()` and verified the fix is real: temporarily
added ROADMAP.md to a class in the live register and confirmed the
test goes red, then restored it clean.

113 -> 116 tests.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01W1ZJFViVYpr8cvf4fs91j1
This commit is contained in:
Kjell Tore Guttormsen 2026-08-04 22:20:36 +02:00
commit aefbadf9a8
2 changed files with 69 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

33
scripts/repo-standard-check.mjs
View file

@ -462,23 +462,46 @@ const BADGE_URL = /shields\.io|badgen\.net|\/badges?[/.]/i;
// cannot carry a hard limit.
const BADGE_INFLECTION = 5;
export function checkBadges({ readme }) {
export function checkBadges({ readme, present }) {
const have = new Set(present ?? []);
const findings = [];
let badgeCount = 0;
for (const line of String(readme ?? '').split('\n')) {
// Any image, any host. Restricting this to img.shields.io would have missed
// a self-hosted SVG asserting exactly the same unverified thing.
for (const m of line.matchAll(/(\[)?!\[([^\]]*)\]\(([^)\s]+)\)(\])?/g)) {
const linked = m[1] === '[' && m[4] === ']';
// The trailing `(?:\]\(target\))?` is the LINK the badge is wrapped in —
// previously unmatched, so being linked at all silently ended scrutiny
// whether or not the link actually went anywhere.
for (const m of line.matchAll(/(\[)?!\[([^\]]*)\]\(([^)\s]+)\)(?:\]\(([^)\s]+)\))?/g)) {
const linkTarget = m[1] === '[' ? m[4] : undefined;
const linked = linkTarget !== undefined;
const label = `${m[2]} ${m[3]}`;
if (BADGE_URL.test(m[3])) badgeCount++;
if (!linked && CLAIM_BADGE.test(label)) {
if (!CLAIM_BADGE.test(label)) continue;
if (!linked) {
findings.push({
level: 'WARN',
code: 'BADGE-STATIC-CLAIM',
bucket: 'weakening',
msg: `static badge asserts a run that nothing verifies: \`${m[2]}\`. A badge like this is a claim dressed as evidence — link it to a real run, or drop it.`,
});
continue;
}
// A linked badge is only as honest as its target. An external target
// (the ordinary case — a CI provider's own page) needs the network to
// verify and is deliberately out of scope, same as checkInternalLinks.
// A relative target this gate CAN check without the network — and a
// relative target that resolves nowhere is worse than a static badge:
// it LOOKS verified.
if (/^[a-z][a-z0-9+.-]*:/i.test(linkTarget)) continue;
const resolved = resolveRelative('README.md', linkTarget.split('#')[0]);
if (resolved !== null && !have.has(resolved)) {
findings.push({
level: 'ERROR',
code: 'BADGE-DEAD-LINK',
bucket: 'broken',
msg: `${m[2]} badge links to \`${linkTarget}\`, which does not resolve — a linked badge pointing nowhere is a claim dressed as evidence, worse than a static one because it looks verified.`,
});
}
}
}
@ -800,7 +823,7 @@ export function classifyRepo(
...checkLinks({ files }, register),
...checkInternalLinks({ files, present }),
...checkLicenseClaim({ readme, present }),
...checkBadges({ readme }),
...checkBadges({ readme, present }),
...checkReadmeLanguage({ readme, name }, register),
...checkBoilerplate({ files }),
...checkVersionConsistency({ pluginVersion, readmeBadge, changelogTop, tags }),

45
scripts/repo-standard-check.test.mjs
View file

@ -33,6 +33,7 @@ import {
fetchWithRetry,
headerLine,
withEngineVersion,
loadRegister,
} from './repo-standard-check.mjs';
const REGISTER = {
@ -374,9 +375,14 @@ test('required files are per class, not flat across the org', () => {
assert.equal(missing.some((f) => f.code === 'FILE-MISSING' && f.msg.includes('LICENSE')), true);
});
// Was tautological: it read `required_files` off the test's OWN local
// REGISTER fixture, so it could only ever check that this file agreed with
// itself — a typo in the real register/repos.json would drift silently past
// it. Reading the live register makes it an actual regression guard.
test('no class requires a ROADMAP — it is 0/18 and belongs to a later step', () => {
for (const klass of Object.keys(REGISTER.classes)) {
assert.equal(REGISTER.classes[klass].required_files.includes('ROADMAP.md'), false);
const live = loadRegister();
for (const klass of Object.keys(live.classes)) {
assert.equal(live.classes[klass].required_files.includes('ROADMAP.md'), false);
}
});
@ -560,6 +566,34 @@ test('a build badge that links to a real run is fine', () => {
assert.equal(checkBadges({ readme }).filter((f) => f.level !== 'OK').length, 0);
});
// The gap: being LINKED was treated as proof of a real run, but nothing ever
// checked that the link actually went anywhere. A badge linked to a dead
// relative path is a claim dressed as evidence that LOOKS more credible than
// a static one, not less. External targets (the ordinary case — a CI
// provider) still need the network and stay out of scope, same as
// checkInternalLinks.
test('a linked badge whose relative target does not exist is a finding, not silently fine', () => {
const readme = '[![Tests](https://img.shields.io/badge/tests-passing-green)](docs/ci-results.md)';
const f = checkBadges({ readme, present: ['README.md'] });
assert.equal(f.some((x) => x.code === 'BADGE-DEAD-LINK'), true);
const hit = f.find((x) => x.code === 'BADGE-DEAD-LINK');
assert.equal(hit.level, 'ERROR');
assert.equal(hit.bucket, 'broken');
});
test('a linked badge whose relative target exists is fine', () => {
const readme = '[![Tests](https://img.shields.io/badge/tests-passing-green)](docs/ci-results.md)';
const f = checkBadges({ readme, present: ['README.md', 'docs/ci-results.md'] });
assert.equal(f.some((x) => x.code === 'BADGE-DEAD-LINK'), false);
});
test('a linked badge with no `present` given is not falsely flagged dead — external targets stay unverifiable', () => {
// Guards the call site: checkBadges must still tolerate being called
// without `present`, same as checkInternalLinks tolerates it.
const readme = '[![CI](https://forge.example/repo/badges/workflows/ci.yml/badge.svg)](https://forge.example/repo/actions)';
assert.equal(checkBadges({ readme }).some((x) => x.code === 'BADGE-DEAD-LINK'), false);
});
// Reported by llm-ingestion-pipeline-security (coord, 2026-08-03): a bare
// `status` badge is a self-declared maturity label ("alpha", "experimental"),
// the same class as version/licence/platform, which already assert no run —
@ -774,9 +808,12 @@ test('the security trait requires limitations to be stated', () => {
assert.equal(f.some((x) => x.code === 'HEADING-MISSING' && x.msg.includes('Known limitations')), true);
});
// Same fix as the ROADMAP test above: was checking the test's own fixture
// against itself. Reading the live register makes it a real guard again.
test('CONTRIBUTING and CODE_OF_CONDUCT are required by no class — the maintainer works alone', () => {
for (const klass of Object.keys(REGISTER.classes)) {
const req = REGISTER.classes[klass].required_files;
const live = loadRegister();
for (const klass of Object.keys(live.classes)) {
const req = live.classes[klass].required_files;
assert.equal(req.includes('CONTRIBUTING.md'), false);
assert.equal(req.includes('CODE_OF_CONDUCT.md'), false);
assert.equal(req.includes('MAINTAINERS.md'), false);