repo-standard/scripts/repo-standard-check.test.mjs
Kjell Tore Guttormsen 1dfd408ffc feat(engine)!: "no version to compare against" is a verdict, not a skip
`VERSION-UNAVAILABLE` (SKIP/notRun) becomes `VERSION-NONE` (OK). A repo that
claims no version in any of the four places one can be written down has not
left a check un-run — the check ran, saw all four, and found no subject. That
is the shape `checkReadmeLanguage` has always answered with OK.

0.8.0 deferred this and recorded the reason in the engine: re-levelling "moves
a repo's status". Measured false across 19 clones — an added OK cannot worsen
the worst *judged* finding, and all three repos emitting the code already read
OK. Status moves only for a repo whose entire finding set was skips.

OK cannot bless a real gap here: no class requires a version file, and a
`plugin` missing its manifest is an independent FILE-MISSING ERROR.

Renamed, not just re-levelled: `-UNAVAILABLE` is this engine's naming for a
notRun skip, and one OK-level `-UNAVAILABLE` would mislead exactly the reader
this change serves. A source scan keeps the old name out.

A/B sweep, both engines offline, 19 clones: 223 judged findings, every status
and buckets identical; 3 conversions (notChecked 14→13, 3→2, 14→13), byDesign
untouched. 170 tests. `--refresh`: register 21 = forge 21.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CTNGvxzCLZp3UXfibYG4kb
2026-08-09 22:05:21 +02:00

1716 lines
82 KiB
JavaScript

// Tests for the repo-standard gate.
//
// The pure classifiers are the unit under test — the I/O shell (inspectRepo/runGate)
// is exercised against a live checkout by the CLI, not here.
//
// The link-check fixtures are the six measured false positives from the census.
// They are the reason this gate has three outcomes instead of a boolean.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import {
countCodepoints,
normalizeRepoRef,
extractOpenRefs,
classifyRef,
checkLinks,
checkDescription,
checkFirstScreen,
checkInstallBlock,
checkRequiredFiles,
checkVersionConsistency,
checkHeadings,
checkBadges,
checkReadmeLanguage,
checkBoilerplate,
checkLicenseClaim,
checkInternalLinks,
resolveRelative,
checkInstallTruth,
classifyRepo,
levelOf,
parseRepoNameFromRemote,
extractChangelogTop,
fetchWithRetry,
headerLine,
withEngineVersion,
readEngineCommit,
loadRegister,
groupSkips,
skipsOf,
} from './repo-standard-check.mjs';
const REGISTER = {
org: 'open',
marketplace: {
name: 'ktg-plugin-marketplace',
url: 'https://git.fromaitochitta.com/open/ktg-plugin-marketplace.git',
},
repos: {
'llm-security': 'plugin',
'repo-mailbox': 'plugin',
'repo-standard': 'plugin',
'ktg-plugin-marketplace': 'catalog',
'playground-design-system': 'shared-asset',
'.profile': 'org-profile',
'llm-ingestion-pipeline-security': 'standalone',
},
non_repos: {
coord: 'Retired repo name, deliberately still alive in prose: the CLI, the mailbox root and CLAUDE_COORD_DIR kept it — they are the transport protocol, not the product.',
_broadcast: 'reserved engine namespace',
'llm-ingestion-guard': 'package name, not a repo',
'claude-code-llm-security': 'pre-split name of llm-security',
},
classes: {
plugin: {
required_files: ['README.md', 'LICENSE', 'CHANGELOG.md', '.claude-plugin/plugin.json'],
required_headings: ['## Install', '## Non-goals', '## Changelog'],
install: 'plugin',
},
catalog: {
required_files: ['README.md', 'LICENSE', 'GOVERNANCE.md', 'CONVENTIONS.md', '.claude-plugin/marketplace.json'],
required_headings: ['## Install', '## Non-goals'],
install: 'catalog',
},
'shared-asset': { required_files: ['README.md', 'LICENSE'], required_headings: ['## Non-goals'], install: 'vendor' },
'org-profile': { required_files: ['README.md'], required_headings: [], install: 'none', readme_desc_match: false },
standalone: { required_files: ['README.md', 'LICENSE'], required_headings: ['## Install', '## Non-goals'], install: 'package' },
},
titles: {
'llm-security': 'LLM Security Plugin for Claude Code',
},
traits: {
'llm-security': ['security'],
'llm-ingestion-pipeline-security': ['security'],
},
trait_requirements: {
security: {
required_files: ['SECURITY.md'],
required_headings: ['## Known limitations'],
},
},
description_max_codepoints: 180,
};
// ---------------------------------------------------------------- measurement
test('countCodepoints measures codepoints, not bytes and not UTF-16 units', () => {
// The em-dash exposes only the byte layer: 3 bytes, 1 codepoint, 1 UTF-16 unit.
assert.equal(countCodepoints('a—b'), 3);
assert.equal(Buffer.byteLength('a—b', 'utf8'), 5);
// 👉 is astral: 1 codepoint but 2 UTF-16 units. This is the layer the em-dash hides.
assert.equal(countCodepoints('👉'), 1);
assert.equal('👉'.length, 2);
});
// ------------------------------------------------------------- ref extraction
test('normalizeRepoRef strips a .git suffix and a trailing slash', () => {
// ~20 "dead" names collapsed to 3 real ones once .git was normalised.
assert.equal(normalizeRepoRef('llm-security.git'), 'llm-security');
assert.equal(normalizeRepoRef('llm-security/'), 'llm-security');
assert.equal(normalizeRepoRef('llm-security'), 'llm-security');
});
test('extractOpenRefs finds names in URL position only', () => {
const text = [
'clone https://git.fromaitochitta.com/open/llm-security.git today',
'see https://git.fromaitochitta.com/open/repo-mailbox/src/branch/main/README.md',
].join('\n');
const names = extractOpenRefs(text).map((r) => r.name);
assert.deepEqual(names, ['llm-security', 'repo-mailbox']);
});
test('extractOpenRefs ignores path position, prose and bare directory names', () => {
// False positives #4, #5, #6 — the text is correct and will STAY correct.
const text = [
'the mailbox root is ~/.claude/coord/_broadcast/inbox/',
'coord is the transport protocol, not the product',
'the catalog lives in ktg-plugin-marketplace/catalog',
'the package llm-ingestion-guard is published from that repo',
].join('\n');
assert.deepEqual(extractOpenRefs(text), []);
});
test('extractOpenRefs ignores an org segment inside an API endpoint path', () => {
// Measured false positive (catalog RUNBOOK.md:39, :114): the Forgejo API
// path /api/v1/orgs/open/repos has "open" as the org argument to the API,
// not a repo reference — "repos" is the literal API resource segment, and
// "open" is not the first path segment after the host.
const text = 'curl -X POST https://git.fromaitochitta.com/api/v1/orgs/open/repos';
assert.deepEqual(extractOpenRefs(text), []);
});
test('extractOpenRefs handles the ssh scp-style form', () => {
const refs = extractOpenRefs('git@git.fromaitochitta.com:open/llm-security.git');
assert.deepEqual(refs.map((r) => r.name), ['llm-security']);
});
test('extractOpenRefs finds a ref on a schemaless host — the scheme is not what makes it a URL', () => {
// Measured false negative (org-ops census 03, llm-security/V3-UPGRADE.md:343):
// a subtree instruction writes the host without a scheme. The name is still
// in URL position — `open` is the first segment after a dotted host — and it
// is still a name the register can resolve. Requiring `://` hid a WARN.
const refs = extractOpenRefs('- Subtree push to `git.fromaitochitta.com/open/claude-code-llm-security`');
assert.deepEqual(refs.map((r) => r.name), ['claude-code-llm-security']);
});
test('the schemaless form does not reopen the API-endpoint false positive', () => {
// The whole point of the host segment excluding `/` survives losing the
// scheme: `orgs` is the segment before `open`, so `open` is not first.
assert.deepEqual(extractOpenRefs('curl git.fromaitochitta.com/api/v1/orgs/open/repos'), []);
});
test('a dotted PATH segment is not a host — only a host puts a name in URL position', () => {
// The guard that keeps the schemaless branch from matching anywhere a dot
// happens to precede `open/`: a path segment is preceded by `/`, a host is not.
assert.deepEqual(extractOpenRefs('see https://git.fromaitochitta.com/docs/v1.2/open/notes'), []);
assert.deepEqual(extractOpenRefs('the fixture lives at test/nav.golden/open/index.md'), []);
});
test('a schemaless host is counted ONCE, not twice, when a scheme is present', () => {
// Two alternatives can match the same text; the leftmost consumes it.
const refs = extractOpenRefs('https://git.fromaitochitta.com/open/llm-security');
assert.deepEqual(refs.map((r) => r.name), ['llm-security']);
});
test('extractOpenRefs reports the 1-indexed line of each hit', () => {
const text = 'line one\nline two\nhttps://git.fromaitochitta.com/open/llm-security';
assert.equal(extractOpenRefs(text)[0].line, 3);
});
// ---------------------------------------------- three outcomes, not a boolean
test('classifyRef separates repo, non-repo and unknown', () => {
assert.equal(classifyRef('llm-security', REGISTER), 'repo');
assert.equal(classifyRef('.profile', REGISTER), 'repo');
assert.equal(classifyRef('coord', REGISTER), 'non-repo');
assert.equal(classifyRef('nonesuch', REGISTER), 'unknown');
});
test('a URL-position ref to a known non-repo is a DISTINCT outcome from no match', () => {
// The specification requirement: "no match" and "match on something that is
// not a repo" must never share an outcome, or the loss goes silent.
const bad = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/nonesuch' } }, REGISTER);
const odd = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/coord' } }, REGISTER);
assert.equal(bad[0].level, 'ERROR');
assert.equal(bad[0].code, 'LINK-DEAD');
assert.equal(odd[0].level, 'WARN');
assert.equal(odd[0].code, 'LINK-NON-REPO');
assert.notEqual(bad[0].code, odd[0].code);
// The reason travels with the finding, so the reader is not sent measuring again.
assert.match(odd[0].msg, /transport protocol/);
});
test('a dead ref names its successor when the register knows one', () => {
const f = checkLinks(
{ files: { 'README.md': 'https://git.fromaitochitta.com/open/claude-code-llm-security' } },
REGISTER,
);
assert.equal(f[0].level, 'WARN');
assert.match(f[0].msg, /pre-split name/);
});
test('the .git suffix does not manufacture a dead reference', () => {
const f = checkLinks(
{ files: { 'README.md': 'https://git.fromaitochitta.com/open/llm-security.git' } },
REGISTER,
);
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
});
test('the hidden .profile resolves — the enumerator sees what a glob misses', () => {
const f = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/.profile' } }, REGISTER);
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
});
test('checkLinks emits OK when every reference resolves — silence is not a pass', () => {
// Measured gap (org-ops census 03b): "no dead references" and "the check
// never ran" produced identical output, so a sweep across 19 repos could not
// tell 19 clean repos from 19 unread ones. The engine already applies "three
// outcomes, never two" to classifyRef; this applies it to its own result.
const f = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/llm-security' } }, REGISTER);
assert.equal(f.length, 1);
assert.equal(f[0].level, 'OK');
assert.equal(f[0].code, 'LINKS-OPEN-REFS');
// The count is the evidence: an OK that cannot say how many it checked is
// the same silence in a different colour.
assert.match(f[0].msg, /1 /);
});
test('a repo with no open/ references at all is still a ran check, not a blank', () => {
const f = checkLinks({ files: { 'README.md': '# title\n\nno references here' } }, REGISTER);
assert.equal(f.length, 1);
assert.equal(f[0].level, 'OK');
assert.match(f[0].msg, /no `open\/` references/);
});
test('checkLinks SKIPs when no files were enumerated — that is the "did not run" case', () => {
const f = checkLinks({ files: {} }, REGISTER);
assert.equal(f.length, 1);
assert.equal(f[0].level, 'SKIP');
assert.equal(f[0].code, 'LINKS-OPEN-REFS-UNAVAILABLE');
});
test('a link whose display text repeats its own URL is ONE reference, not two', () => {
// Introduced by the schemaless branch and measured before shipping it
// (llm-security/V3-ANNOUNCEMENT.md:124): `[host/open/x](https://host/open/x)`
// matches on both halves. For a dead name that is one defect printed twice;
// for a live one it inflates the very count the OK line offers as evidence.
const line = '[git.fromaitochitta.com/open/nonesuch](https://git.fromaitochitta.com/open/nonesuch)';
const f = checkLinks({ files: { 'README.md': line } }, REGISTER);
assert.equal(f.length, 1);
assert.equal(f[0].code, 'LINK-DEAD');
});
test('the OK count does not double-count a self-linking URL', () => {
const line = '[git.fromaitochitta.com/open/llm-security](https://git.fromaitochitta.com/open/llm-security)';
const f = checkLinks({ files: { 'README.md': line } }, REGISTER);
assert.equal(f[0].level, 'OK');
assert.match(f[0].msg, /^1 /);
});
test('two DIFFERENT names on one line stay two references', () => {
// The dedup key is name-and-line, not line — collapsing by line alone would
// hide the second dead name behind the first.
const line = 'https://git.fromaitochitta.com/open/nonesuch vs https://git.fromaitochitta.com/open/alsonone';
const f = checkLinks({ files: { 'README.md': line } }, REGISTER);
assert.equal(f.length, 2);
});
test('the same name on two DIFFERENT lines stays two references', () => {
const text = 'https://git.fromaitochitta.com/open/nonesuch\nand again https://git.fromaitochitta.com/open/nonesuch';
const f = checkLinks({ files: { 'README.md': text } }, REGISTER);
assert.equal(f.length, 2);
});
test('checkLinks prints no OK beside a finding, not even a WARN-only one', () => {
const dead = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/nonesuch' } }, REGISTER);
const warn = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/coord' } }, REGISTER);
assert.equal(dead.filter((x) => x.level === 'OK').length, 0);
assert.equal(warn.filter((x) => x.level === 'OK').length, 0);
});
// ------------------------------------------------------------- description
test('description length is bounded, measured in codepoints', () => {
assert.equal(checkDescription('a fine description', REGISTER)[0].level, 'OK');
assert.equal(checkDescription('', REGISTER)[0].level, 'ERROR');
assert.equal(checkDescription('x'.repeat(181), REGISTER)[0].level, 'ERROR');
assert.equal(checkDescription('x'.repeat(180), REGISTER)[0].level, 'OK');
});
test('an unavailable description is SKIP, never a pass', () => {
// Offline is not compliance. A check that could not run says so.
const f = checkDescription(null, REGISTER);
assert.equal(f[0].level, 'SKIP');
});
// ------------------------------------------------------------- first screen
test('README line 1 must be the H1, and the description line must match the forge', () => {
const readme = '# repo-mailbox\nA local mailbox for coordination.\n';
assert.equal(
checkFirstScreen({ readme, name: 'repo-mailbox', description: 'A local mailbox for coordination.' }, REGISTER)
.every((f) => f.level === 'OK'),
true,
);
});
test('a README whose opening line diverges from the description is an ERROR', () => {
const readme = '# repo-mailbox\nSomething else entirely.\n';
const f = checkFirstScreen({ readme, name: 'repo-mailbox', description: 'A local mailbox for coordination.' }, REGISTER);
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'README-DESC'), true);
});
test('first-screen description match is SKIP when the forge text is unavailable', () => {
const f = checkFirstScreen({ readme: '# x\nbody\n', name: 'x', description: null }, REGISTER);
assert.equal(f.some((x) => x.code === 'README-DESC' && x.level === 'SKIP'), true);
});
test('a missing H1 is an ERROR', () => {
const f = checkFirstScreen({ readme: 'no heading here\n', name: 'x', description: null }, REGISTER);
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'README-H1'), true);
});
test('an H1 that differs from the repo name is a WARN, not a failure', () => {
// `# OKR for Public Sector` is a naming choice, not a defect: the thread that
// must hold is description == catalog == opening line, and the H1 is none of
// those three. Surface it; let the operator decide.
const f = checkFirstScreen({ readme: '# OKR for Public Sector\nbody\n', name: 'okr', description: null }, REGISTER);
assert.equal(f.some((x) => x.level === 'WARN' && x.code === 'README-H1'), true);
assert.equal(f.some((x) => x.level === 'ERROR'), false);
});
test('a differing H1 does not stop the description check from running', () => {
const f = checkFirstScreen({ readme: '# Nice Title\nthe description\n', name: 'x', description: 'the description' }, REGISTER);
assert.equal(f.some((x) => x.code === 'README-DESC' && x.level === 'OK'), true);
});
// ------------------------------------------------- registered README titles
// Census 05 measured the same 6 README-H1 WARNs three rounds running, because
// a decided YES had nowhere to live. `titles` is that place: with an entry the
// H1 is OK, without one it WARNs exactly as before — so "we decided this" and
// "nobody has looked at it" stop sharing one outcome. The wanted side effect is
// that a repo nobody has ruled on stands alone once the others are registered.
test('an H1 matching the registered title is OK, not a WARN', () => {
const readme = '# LLM Security Plugin for Claude Code\nbody\n';
const f = checkFirstScreen({ readme, name: 'llm-security', description: null }, REGISTER);
assert.equal(f.some((x) => x.code === 'README-H1' && x.level === 'OK'), true);
assert.equal(f.some((x) => x.code === 'README-H1' && x.level === 'WARN'), false);
});
test('a repo with no registered title still WARNs — deciding and not looking stay different outcomes', () => {
const f = checkFirstScreen({ readme: '# Interaction Awareness\nbody\n', name: 'repo-mailbox', description: null }, REGISTER);
assert.equal(f.some((x) => x.code === 'README-H1' && x.level === 'WARN'), true);
});
test('an H1 matching neither the name nor the registered title WARNs, and the message names both', () => {
const f = checkFirstScreen({ readme: '# Something Drifted\nbody\n', name: 'llm-security', description: null }, REGISTER);
const warn = f.find((x) => x.code === 'README-H1' && x.level === 'WARN');
assert.ok(warn, 'expected a README-H1 WARN');
assert.match(warn.msg, /Something Drifted/);
assert.match(warn.msg, /LLM Security Plugin for Claude Code/);
});
test('the repo name still passes when a title is registered — both spellings are accepted', () => {
const f = checkFirstScreen({ readme: '# llm-security\nbody\n', name: 'llm-security', description: null }, REGISTER);
assert.equal(f.some((x) => x.code === 'README-H1' && x.level === 'OK'), true);
});
// --------------------------------------------- org-profile README-DESC exemption
// For an ordinary repo the README opening and the forge description describe the
// SAME subject, and equality is the right demand. For `org-profile` they do not:
// the README is the organisation's landing page, the forge description describes
// the repo. Both texts are correct about their own subject. The description field
// also renders as PLAIN TEXT, so a landing page's opening link could only ever
// match by putting raw markdown on a real surface.
const PROFILE_README = '# .profile\nCompanion repos, plugins, and tools for [From AI to Chitta](https://fromaitochitta.com).\n';
test('org-profile is exempt from the README-DESC equality check', () => {
const f = checkFirstScreen(
{ readme: PROFILE_README, name: '.profile', klass: 'org-profile', description: 'Organization profile and navigation for the open org.' },
REGISTER,
);
const desc = f.find((x) => x.code === 'README-DESC');
assert.ok(desc, 'expected a README-DESC finding — an exemption must still be recorded');
assert.equal(desc.level, 'OK');
assert.equal(f.some((x) => x.level === 'ERROR'), false);
});
test('the exemption is per class, not global — the same README under a plugin class is still an ERROR', () => {
const f = checkFirstScreen(
{ readme: PROFILE_README, name: 'llm-security', klass: 'plugin', description: 'Organization profile and navigation for the open org.' },
REGISTER,
);
assert.equal(f.some((x) => x.code === 'README-DESC' && x.level === 'ERROR'), true);
});
test('the exemption does not silence the H1 check for org-profile', () => {
const f = checkFirstScreen(
{ readme: 'no heading here\n', name: '.profile', klass: 'org-profile', description: 'anything' },
REGISTER,
);
assert.equal(f.some((x) => x.code === 'README-H1' && x.level === 'ERROR'), true);
});
// ------------------------------------------------------------ install block
const MKT = REGISTER.marketplace;
test('plugin install needs BOTH lines: marketplace add and a CLI install command', () => {
const readme = [
'## Install',
'```',
`claude plugin marketplace add ${MKT.url}`,
`claude plugin install repo-mailbox@${MKT.name}`,
'```',
].join('\n');
const f = checkInstallBlock({ readme, name: 'repo-mailbox', klass: 'plugin' }, REGISTER);
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
});
test('the slash form counts as the CLI command', () => {
const readme = [
'## Install',
`claude plugin marketplace add ${MKT.url}`,
`/plugin install claude-design@${MKT.name}`,
].join('\n');
const f = checkInstallBlock({ readme, name: 'claude-design', klass: 'plugin' }, REGISTER);
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
});
test('enabledPlugins JSON is an ALLOWED ADDITION, never a replacement for the CLI command', () => {
// This is the corrected defect A: 7 of 11 plugin READMEs stop after
// `marketplace add`. The JSON form works and stands in 10 of 11 — what is
// missing is a CLI command, so the contract requires the command and permits
// the JSON alongside it.
const jsonOnly = [
'## Install',
`claude plugin marketplace add ${MKT.url}`,
'Or enable directly in `~/.claude/settings.json`:',
`"enabledPlugins": { "llm-security@${MKT.name}": true }`,
].join('\n');
const f = checkInstallBlock({ readme: jsonOnly, name: 'llm-security', klass: 'plugin' }, REGISTER);
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'INSTALL-NO-CLI'), true);
const both = jsonOnly + `\nclaude plugin install llm-security@${MKT.name}\n`;
const g = checkInstallBlock({ readme: both, name: 'llm-security', klass: 'plugin' }, REGISTER);
assert.equal(g.filter((x) => x.level === 'ERROR').length, 0);
});
test('a missing marketplace add is its own finding — okr and claude-design are opposite halves', () => {
const noAdd = ['## Install', `/plugin install claude-design@${MKT.name}`].join('\n');
const f = checkInstallBlock({ readme: noAdd, name: 'claude-design', klass: 'plugin' }, REGISTER);
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'INSTALL-NO-MARKETPLACE'), true);
// okr: neither line. Both findings fire — a rule saying "both lines" must not
// hit this repo blind.
const neither = ['## Install', `"enabledPlugins": { "okr@${MKT.name}": true }`].join('\n');
const g = checkInstallBlock({ readme: neither, name: 'okr', klass: 'plugin' }, REGISTER);
assert.equal(g.some((x) => x.code === 'INSTALL-NO-MARKETPLACE'), true);
assert.equal(g.some((x) => x.code === 'INSTALL-NO-CLI'), true);
});
test('the install target must name THIS repo, not another plugin', () => {
const readme = [
'## Install',
`claude plugin marketplace add ${MKT.url}`,
`claude plugin install some-other-plugin@${MKT.name}`,
].join('\n');
const f = checkInstallBlock({ readme, name: 'repo-standard', klass: 'plugin' }, REGISTER);
assert.equal(f.some((x) => x.code === 'INSTALL-NO-CLI'), true);
});
test('ssh in the marketplace add line is an ERROR — marketplace add rejects it', () => {
// Measured end-to-end: `marketplace add ssh://...` → "Invalid git URL".
// The forge UI's clone button hands you exactly that URL.
const readme = [
'## Install',
'claude plugin marketplace add ssh://git@git.fromaitochitta.com/open/ktg-plugin-marketplace.git',
`claude plugin install repo-standard@${MKT.name}`,
].join('\n');
const f = checkInstallBlock({ readme, name: 'repo-standard', klass: 'plugin' }, REGISTER);
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'INSTALL-SSH'), true);
});
test('the install block is parametric — a different marketplace passes on its own values', () => {
// wiki-advise lives on the private ktg/ namespace and is distributed via
// `ktg-privat`. A skill that hardcodes the public marketplace produces an
// install line that does not work there — and being public itself, this skill
// cannot carry private marketplace names.
const priv = {
...REGISTER,
marketplace: { name: 'ktg-privat', url: 'https://git.fromaitochitta.com/ktg/ktg-privat.git' },
};
const readme = [
'## Install',
'claude plugin marketplace add https://git.fromaitochitta.com/ktg/ktg-privat.git',
'claude plugin install wiki-advise@ktg-privat',
].join('\n');
const f = checkInstallBlock({ readme, name: 'wiki-advise', klass: 'plugin' }, priv);
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
});
test('the catalog needs only marketplace add — it IS the marketplace', () => {
const readme = `## Install\nclaude plugin marketplace add ${MKT.url}`;
const f = checkInstallBlock({ readme, name: 'ktg-plugin-marketplace', klass: 'catalog' }, REGISTER);
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
});
test('a shared asset is vendored, not installed — a plugin install line is wrong there', () => {
const asPlugin = `## Install\nclaude plugin install playground-design-system@${MKT.name}`;
const f = checkInstallBlock(
{ readme: asPlugin, name: 'playground-design-system', klass: 'shared-asset' },
REGISTER,
);
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'INSTALL-WRONG-FORM'), true);
});
test('.profile needs no install section at all', () => {
const f = checkInstallBlock({ readme: '# .profile\nOrg profile.\n', name: '.profile', klass: 'org-profile' }, REGISTER);
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
});
// ----------------------------------------------------------- required files
test('required files are per class, not flat across the org', () => {
const ok = checkRequiredFiles({ present: ['README.md'], klass: 'org-profile' }, REGISTER);
assert.equal(ok.filter((f) => f.level === 'ERROR').length, 0);
const missing = checkRequiredFiles({ present: ['README.md'], klass: 'plugin' }, REGISTER);
assert.equal(missing.some((f) => f.code === 'FILE-MISSING' && f.msg.includes('LICENSE')), true);
});
// Was tautological: it read `required_files` off the test's OWN local
// REGISTER fixture, so it could only ever check that this file agreed with
// itself — a typo in the real register/repos.json would drift silently past
// it. Reading the live register makes it an actual regression guard.
test('no class requires a ROADMAP — it is 0/18 and belongs to a later step', () => {
const live = loadRegister();
for (const klass of Object.keys(live.classes)) {
assert.equal(live.classes[klass].required_files.includes('ROADMAP.md'), false);
}
});
// A register row whose class does not exist reads as REGISTERED but runs the
// same zero checks as an unregistered one — the silent-loss shape this gate is
// built to catch, hiding inside a file that looks maintained. Reads the live
// register so every future row is guarded, not just today's.
test('every registered repo names a class that actually exists', () => {
const live = loadRegister();
const classes = Object.keys(live.classes);
for (const [repo, klass] of Object.entries(live.repos)) {
assert.equal(classes.includes(klass), true, `${repo} is registered as unknown class \`${klass}\``);
}
});
// -------------------------------------------------------------- aggregation
test('levelOf ranks ERROR above WARN above OK', () => {
assert.equal(levelOf([{ level: 'OK' }, { level: 'WARN' }, { level: 'ERROR' }]), 'ERROR');
assert.equal(levelOf([{ level: 'OK' }, { level: 'WARN' }]), 'WARN');
assert.equal(levelOf([{ level: 'OK' }]), 'OK');
});
// The defect this replaced: SKIP outranked OK, so a repo with 0 ERROR, 0 WARN
// and a dozen OK headlined as "skipped". Reported by org-ops against five repos
// in census 05; `okr` has the most OK in the org and read as unread. SKIP is not
// a severity — it is the absence of a judgement, and it cannot be the worst of a
// set that contains real ones.
test('a SKIP does not outrank an OK — it is absence of judgement, not a severity', () => {
assert.equal(levelOf([{ level: 'OK' }, { level: 'SKIP' }]), 'OK');
assert.equal(levelOf([{ level: 'SKIP' }, { level: 'WARN' }]), 'WARN');
assert.equal(levelOf([{ level: 'SKIP' }, { level: 'ERROR' }]), 'ERROR');
});
// The other half of the same rule, and the half that keeps "`SKIP` is never a
// pass" true: when there is nothing else to be worst OF, SKIP stands.
test('SKIP still wins when nothing was judged at all', () => {
assert.equal(levelOf([{ level: 'SKIP' }]), 'SKIP');
assert.equal(levelOf([{ level: 'SKIP' }, { level: 'SKIP' }]), 'SKIP');
});
// An empty finding set means no check ran, which is the same state as an
// all-SKIP set — not a clean bill. It cannot occur for a registered repo, so
// this pins a deliberate choice rather than an observed case.
test('no findings at all is SKIP, not OK — nothing ran', () => {
assert.equal(levelOf([]), 'SKIP');
});
test('an unregistered repo is SKIP, not a pass — the gate refuses to guess a class', () => {
const r = classifyRepo({ name: 'stranger', files: {}, present: [], description: null }, REGISTER);
assert.equal(r.status, 'SKIP');
assert.equal(r.notChecked, 1);
});
// Coverage is its own axis, carried explicitly rather than left for each
// consumer to re-derive from `findings`. An exemption nobody can see reads
// exactly like a check that silently stopped running.
test('notChecked counts the skipped checks beside the status, on every result', () => {
const r = classifyRepo({ name: 'stranger', files: {}, present: [], description: null }, REGISTER);
assert.equal(typeof r.notChecked, 'number');
});
test('a fully compliant plugin repo classifies OK', () => {
const readme = [
'# repo-mailbox',
'A local mailbox for coordination.',
'',
'Body text.',
'',
'![Version](https://img.shields.io/badge/version-0.7.0-blue)',
'',
'## Install',
`claude plugin marketplace add ${MKT.url}`,
`claude plugin install repo-mailbox@${MKT.name}`,
'',
'## Non-goals',
'It is not a state store.',
'',
'## Changelog',
'See [CHANGELOG.md](CHANGELOG.md).',
].join('\n');
const r = classifyRepo(
{
name: 'repo-mailbox',
files: { 'README.md': readme },
present: ['README.md', 'LICENSE', 'CHANGELOG.md', '.claude-plugin/plugin.json'],
description: 'A local mailbox for coordination.',
pluginVersion: '0.7.0',
readmeBadge: '0.7.0',
changelogTop: '0.7.0',
tags: ['v0.7.0'],
catalogNames: ['repo-mailbox'],
},
REGISTER,
);
assert.equal(r.status, 'OK');
assert.deepEqual(r.buckets, { broken: 0, missing: 0, weakening: 0 });
});
test('the security trait travels from the register into the verdict', () => {
const r = classifyRepo(
{
name: 'llm-security',
files: { 'README.md': '# llm-security\ndesc\n## Install\n## Non-goals\n## Changelog\n' },
present: ['README.md', 'LICENSE', 'CHANGELOG.md', '.claude-plugin/plugin.json'],
description: 'desc',
},
REGISTER,
);
assert.deepEqual(r.traits, ['security']);
assert.equal(r.findings.some((f) => f.code === 'FILE-MISSING' && f.msg.includes('SECURITY.md')), true);
assert.equal(r.findings.some((f) => f.code === 'HEADING-MISSING' && f.msg.includes('Known limitations')), true);
});
// =========================================================================
// The brief's reporting contract: findings carry a BUCKET alongside a level.
// "broken now" (strangers are blocked or misled), "missing" (an expected
// artefact is absent), "weakening" (present, but reads as amateur). The two
// axes are independent: a weakening finding can still be an ERROR.
// =========================================================================
test('every ERROR and WARN finding carries a bucket', () => {
const readme = ['## Install', `"enabledPlugins": { "okr@${MKT.name}": true }`].join('\n');
const all = [
...checkInstallBlock({ readme, name: 'okr', klass: 'plugin' }, REGISTER),
...checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/nonesuch' } }, REGISTER),
...checkRequiredFiles({ present: ['README.md'], klass: 'plugin' }, REGISTER),
...checkDescription('', REGISTER),
];
const graded = all.filter((f) => f.level === 'ERROR' || f.level === 'WARN');
assert.ok(graded.length > 0);
for (const f of graded) {
assert.ok(['broken', 'missing', 'weakening'].includes(f.bucket), `${f.code} has bucket ${f.bucket}`);
}
});
test('an unusable install path is `broken`, an absent file is `missing`', () => {
const readme = ['## Install', 'nothing useful here'].join('\n');
const inst = checkInstallBlock({ readme, name: 'okr', klass: 'plugin' }, REGISTER);
assert.equal(inst.find((f) => f.code === 'INSTALL-NO-CLI').bucket, 'broken');
const files = checkRequiredFiles({ present: ['README.md'], klass: 'plugin' }, REGISTER);
assert.equal(files.find((f) => f.code === 'FILE-MISSING').bucket, 'missing');
});
// ------------------------------------------------------- version consistency
test('version consistency: manifest, README badge and CHANGELOG must agree', () => {
const ok = checkVersionConsistency({ pluginVersion: '0.1.0', readmeBadge: '0.1.0', changelogTop: '0.1.0', tags: ['v0.1.0'] });
assert.equal(ok.filter((f) => f.level === 'ERROR').length, 0);
const drift = checkVersionConsistency({ pluginVersion: '0.1.0', readmeBadge: '0.0.9', changelogTop: '0.1.0', tags: ['v0.1.0'] });
assert.equal(drift.some((f) => f.level === 'ERROR' && f.code === 'VERSION-BADGE'), true);
const stale = checkVersionConsistency({ pluginVersion: '0.2.0', readmeBadge: '0.2.0', changelogTop: '0.1.0', tags: ['v0.1.0', 'v0.2.0'] });
assert.equal(stale.some((f) => f.level === 'ERROR' && f.code === 'VERSION-CHANGELOG'), true);
});
test('an untagged repo SKIPs the tag comparison rather than failing it', () => {
// Nothing has been released yet. That is a state, not a defect — and "SKIP is
// never a pass" means it must say so rather than quietly succeed.
const f = checkVersionConsistency({ pluginVersion: '0.1.0', readmeBadge: '0.1.0', changelogTop: '0.1.0', tags: [] });
assert.equal(f.some((x) => x.code === 'VERSION-TAG' && x.level === 'SKIP'), true);
assert.equal(f.some((x) => x.level === 'ERROR'), false);
});
test('a released version with no matching tag is an ERROR', () => {
const f = checkVersionConsistency({ pluginVersion: '0.3.1', readmeBadge: '0.3.1', changelogTop: '0.3.1', tags: ['v0.1.0'] });
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'VERSION-TAG'), true);
});
// Reported by llm-ingestion-okf (coord, 2026-08-03): a PEP 440 pre-release
// (`0.5.0a2`) matches the manifest and the tag exactly, but the CHANGELOG
// extractor truncated it to `0.5.0` — the only way to reach 0 ERROR would have
// been to announce a release that never happened.
test('extractChangelogTop keeps a PEP 440 pre-release suffix, not just X.Y.Z', () => {
assert.equal(extractChangelogTop('## [0.5.0a2] - 2026-07-31'), '0.5.0a2');
assert.equal(extractChangelogTop('## [0.1.1] — 2026-08-03'), '0.1.1');
});
test('a PEP 440 pre-release version agrees with its own CHANGELOG heading', () => {
const f = checkVersionConsistency({
pluginVersion: '0.5.0a2', readmeBadge: '0.5.0a2', changelogTop: extractChangelogTop('## [0.5.0a2] - 2026-07-31'), tags: ['v0.5.0a2'],
});
assert.equal(f.some((x) => x.level === 'ERROR'), false);
});
// -------------------------------------------------------- required headings
test('required headings are per class — Non-goals is required, not optional', () => {
const full = '# x\n## Install\n## Non-goals\n## Changelog\n';
assert.equal(checkHeadings({ readme: full, klass: 'plugin' }, REGISTER).filter((f) => f.level === 'ERROR').length, 0);
const noNonGoals = '# x\n## Install\n## Changelog\n';
const f = checkHeadings({ readme: noNonGoals, klass: 'plugin' }, REGISTER);
assert.equal(f.some((x) => x.code === 'HEADING-MISSING' && x.msg.includes('Non-goals')), true);
assert.equal(f.find((x) => x.code === 'HEADING-MISSING').bucket, 'missing');
});
test('org-profile requires no headings at all', () => {
const f = checkHeadings({ readme: '# .profile\nprofile\n', klass: 'org-profile' }, REGISTER);
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
});
// ------------------------------------------------------------ badge honesty
test('a static badge asserting test or build status is a finding', () => {
// "tests: 642 passing" as a static image is a claim dressed as evidence.
const f = checkBadges({ readme: '![Tests](https://img.shields.io/badge/tests-34-green)' });
assert.equal(f.some((x) => x.code === 'BADGE-STATIC-CLAIM'), true);
assert.equal(f.find((x) => x.code === 'BADGE-STATIC-CLAIM').bucket, 'weakening');
});
test('static version, licence and platform badges are fine — they assert no run', () => {
const readme = [
'![Version](https://img.shields.io/badge/version-0.1.0-blue)',
'![License](https://img.shields.io/badge/license-MIT-lightgrey)',
'![Platform](https://img.shields.io/badge/platform-Claude_Code_Plugin-purple)',
].join('\n');
assert.equal(checkBadges({ readme }).filter((f) => f.level !== 'OK').length, 0);
});
test('a build badge that links to a real run is fine', () => {
const readme = '[![CI](https://forge.example/repo/badges/workflows/ci.yml/badge.svg)](https://forge.example/repo/actions)';
assert.equal(checkBadges({ readme }).filter((f) => f.level !== 'OK').length, 0);
});
// The gap: being LINKED was treated as proof of a real run, but nothing ever
// checked that the link actually went anywhere. A badge linked to a dead
// relative path is a claim dressed as evidence that LOOKS more credible than
// a static one, not less. External targets (the ordinary case — a CI
// provider) still need the network and stay out of scope, same as
// checkInternalLinks.
test('a linked badge whose relative target does not exist is a finding, not silently fine', () => {
const readme = '[![Tests](https://img.shields.io/badge/tests-passing-green)](docs/ci-results.md)';
const f = checkBadges({ readme, present: ['README.md'] });
assert.equal(f.some((x) => x.code === 'BADGE-DEAD-LINK'), true);
const hit = f.find((x) => x.code === 'BADGE-DEAD-LINK');
assert.equal(hit.level, 'ERROR');
assert.equal(hit.bucket, 'broken');
});
test('a linked badge whose relative target exists is fine', () => {
const readme = '[![Tests](https://img.shields.io/badge/tests-passing-green)](docs/ci-results.md)';
const f = checkBadges({ readme, present: ['README.md', 'docs/ci-results.md'] });
assert.equal(f.some((x) => x.code === 'BADGE-DEAD-LINK'), false);
});
test('a linked badge with no `present` given is not falsely flagged dead — external targets stay unverifiable', () => {
// Guards the call site: checkBadges must still tolerate being called
// without `present`, same as checkInternalLinks tolerates it.
const readme = '[![CI](https://forge.example/repo/badges/workflows/ci.yml/badge.svg)](https://forge.example/repo/actions)';
assert.equal(checkBadges({ readme }).some((x) => x.code === 'BADGE-DEAD-LINK'), false);
});
// Reported by llm-ingestion-pipeline-security (coord, 2026-08-03): a bare
// `status` badge is a self-declared maturity label ("alpha", "experimental"),
// the same class as version/licence/platform, which already assert no run —
// not a run claim like "build status" or "CI status".
test('a static maturity-status badge asserts no run, unlike build/CI status', () => {
const f = checkBadges({ readme: '![Status](https://img.shields.io/badge/status-alpha-orange)' });
assert.equal(f.some((x) => x.code === 'BADGE-STATIC-CLAIM'), false);
});
test('a build- or CI-status badge is still caught — only bare "status" was too wide', () => {
const build = checkBadges({ readme: '![Build Status](https://img.shields.io/badge/build-passing-green)' });
assert.equal(build.some((x) => x.code === 'BADGE-STATIC-CLAIM'), true);
const ci = checkBadges({ readme: '![CI Status](https://img.shields.io/badge/ci-passing-green)' });
assert.equal(ci.some((x) => x.code === 'BADGE-STATIC-CLAIM'), true);
});
// ----------------------------------------------------------- badge crowding
// Trockman et al., ICSE 2018 (n=294,941 npm packages) measured a non-linear
// relationship between badge count and popularity with a predicted inflection
// at five, motivated by survey respondents calling over-badged READMEs
// cluttered and "trying too hard". The coefficient sits in an appendix with no
// CI or p-value, so this is a WARN and the threshold is the measured inflection
// — not a rounder number that would read as invented.
const badges = (n, wrap = (s) => s) =>
Array.from({ length: n }, (_, i) => wrap(`![B${i}](https://img.shields.io/badge/b${i}-x-blue)`)).join('\n');
test('more than five badges is a finding — the measured inflection point', () => {
const f = checkBadges({ readme: badges(6) });
const hit = f.find((x) => x.code === 'BADGE-COUNT');
assert.ok(hit, 'six badges should produce BADGE-COUNT');
assert.equal(hit.level, 'WARN');
assert.equal(hit.bucket, 'weakening');
});
test('exactly five badges is not a finding — the source cannot carry a harder rule', () => {
assert.equal(checkBadges({ readme: badges(5) }).some((x) => x.code === 'BADGE-COUNT'), false);
});
test('a linked badge still counts toward the total — linking answers honesty, not clutter', () => {
const readme = badges(6, (s) => `[${s}](https://forge.example/run)`);
assert.equal(checkBadges({ readme }).some((x) => x.code === 'BADGE-COUNT'), true);
});
test('content images are not badges, however many there are', () => {
const readme = Array.from({ length: 9 }, (_, i) => `![Architecture ${i}](docs/img/arch${i}.png)`).join('\n');
assert.equal(checkBadges({ readme }).some((x) => x.code === 'BADGE-COUNT'), false);
});
// ---------------------------------------------------------- README language
// The operator owns this axis, exactly as they own `traits`. English is the
// default; a repo aimed ONLY at a Norwegian readership is declared `nb` and is
// then WRONG in English, not right. Measured 2026-08-03: `ms-ai-architect` and
// `okr` are the two such repos, and both currently carry English prose.
const LOCALE_REGISTER = { ...REGISTER, locales: { okr: 'nb' } };
const NB_PROSE = [
'# okr',
'',
'Dette er en plugin som ikke gjør noe annet enn å måle mål og resultater.',
'Den kan kjøres fra Claude Code, og den skal være tilgjengelig når du',
'trenger den. Hvis du vil ha mer, se dokumentasjonen. Alle kommandoer',
'blir kjørt fra en aktiv sesjon, og ingenting av dette krever en server.',
].join('\n');
const EN_PROSE = [
'# okr',
'',
'This is a plugin that does nothing other than measure objectives and',
'results. It can be run from Claude Code, and it is always available when',
'you need it. If you want more, see the documentation. All of the commands',
'are run from an active session, and none of this requires a server.',
].join('\n');
test('a repo declared Norwegian that ships English prose is a finding', () => {
const f = checkReadmeLanguage({ readme: EN_PROSE, name: 'okr' }, LOCALE_REGISTER);
const hit = f.find((x) => x.code === 'README-LANGUAGE');
assert.ok(hit, 'declared nb + English prose should produce README-LANGUAGE');
assert.equal(hit.level, 'WARN');
assert.equal(hit.bucket, 'weakening');
});
test('a repo declared Norwegian that ships Norwegian prose passes', () => {
const f = checkReadmeLanguage({ readme: NB_PROSE, name: 'okr' }, LOCALE_REGISTER);
assert.equal(f.some((x) => x.code === 'README-LANGUAGE'), false);
});
test('an undeclared repo defaults to English, so Norwegian prose is the finding', () => {
const f = checkReadmeLanguage({ readme: NB_PROSE, name: 'repo-standard' }, LOCALE_REGISTER);
assert.equal(f.some((x) => x.code === 'README-LANGUAGE'), true);
});
test('an undeclared repo shipping English prose passes', () => {
const f = checkReadmeLanguage({ readme: EN_PROSE, name: 'repo-standard' }, LOCALE_REGISTER);
assert.equal(f.some((x) => x.code === 'README-LANGUAGE'), false);
});
// Same discipline as the link and boilerplate checks: a Norwegian identifier in
// a shell example must not decide what language the DOCUMENT is written in.
test('code blocks do not decide the language', () => {
const readme = [
EN_PROSE,
'',
'```bash',
'kjør --og --ikke --som --det --den --er --på --til --av --med --om',
'kjør --har --kan --skal --blir --etter --når --også --hvis --eller',
'```',
].join('\n');
const f = checkReadmeLanguage({ readme, name: 'repo-standard' }, LOCALE_REGISTER);
assert.equal(f.some((x) => x.code === 'README-LANGUAGE'), false);
});
// A README with no running prose has no language to be wrong about — the check
// ran and found nothing, which is not the same as a check that could not run.
// Routing it to SKIP would mean no terse repo could ever classify OK.
test('no running prose is an OK, not a SKIP — nothing claims a language', () => {
const f = checkReadmeLanguage({ readme: '# thing\n\nA tool.\n', name: 'repo-standard' }, LOCALE_REGISTER);
assert.equal(f.some((x) => x.code === 'README-LANGUAGE'), false);
assert.equal(f[0].level, 'OK');
});
// A document with real prose that the gate genuinely cannot call IS a SKIP —
// here the question is live and unanswered, unlike the empty case above.
test('prose that mixes languages too evenly is a SKIP, never a pass', () => {
const readme = [NB_PROSE, EN_PROSE].join('\n\n');
const f = checkReadmeLanguage({ readme, name: 'repo-standard' }, LOCALE_REGISTER);
const hit = f.find((x) => x.code === 'README-LANGUAGE-UNDECIDABLE');
assert.ok(hit, 'an evenly bilingual README cannot be called');
assert.equal(hit.level, 'SKIP');
});
// -------------------------------------------------------------- boilerplate
test('unfinished template text is a finding', () => {
const f = checkBoilerplate({ files: { 'README.md': 'Install your-project-name today' } });
assert.equal(f.some((x) => x.code === 'BOILERPLATE'), true);
const g = checkBoilerplate({ files: { 'CODE_OF_CONDUCT.md': 'Report to [INSERT EMAIL ADDRESS]' } });
assert.equal(g.some((x) => x.code === 'BOILERPLATE'), true);
});
test('a lone FIXME with no TODO alongside is still caught', () => {
const f = checkBoilerplate({ files: { 'NOTES.md': 'FIXME: handle the null case here.' } });
assert.equal(f.some((x) => x.level === 'WARN'), true);
});
// Reported by config-audit (coord, 2026-08-03): a scanner whose JOB is to find
// TODO/FIXME markers in OTHER repos names its own detection target in its own
// docs — prose and a table row, neither wrapped in backticks. "TODO/FIXME"
// named together is the convention itself, not a forgotten instance of one.
test('"TODO/FIXME" named together as the convention is not a live marker', () => {
const listItem = checkBoilerplate({
files: { 'agents/scanner-agent.md': "- Flag TODO/FIXME markers that haven't been addressed" },
});
assert.equal(listItem.some((x) => x.level === 'WARN'), false);
const tableRow = checkBoilerplate({
files: { 'knowledge/anti-patterns.md': '| 5 | TODO/FIXME comments in CLAUDE.md | CA-CML-005 | low |' },
});
assert.equal(tableRow.some((x) => x.level === 'WARN'), false);
});
test('ordinary prose is not boilerplate', () => {
const f = checkBoilerplate({ files: { 'README.md': 'This project solves a real problem.' } });
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
});
// --------------------------------------------------- licence claim vs. file
test('a README that cites a LICENSE the repo does not have is broken', () => {
const f = checkLicenseClaim({ readme: 'Released under the [MIT licence](LICENSE).', present: ['README.md'] });
assert.equal(f.some((x) => x.code === 'LICENSE-CLAIMED-ABSENT' && x.bucket === 'broken'), true);
});
test('a cited LICENSE that exists passes', () => {
const f = checkLicenseClaim({ readme: 'See [LICENSE](LICENSE).', present: ['README.md', 'LICENSE'] });
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
});
// ------------------------------------------------------------ internal links
test('a relative link to a file that does not exist is a finding', () => {
const f = checkInternalLinks(
{ files: { 'README.md': 'See [the design](docs/design.md).' }, present: ['README.md'] },
);
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), true);
});
test('relative links that resolve, anchors and external URLs are left alone', () => {
const f = checkInternalLinks({
files: { 'README.md': '[a](docs/design.md) [b](#section) [c](https://example.com) [d](mailto:x@y.z)' },
present: ['README.md', 'docs/design.md'],
});
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
});
// ----------------------------------------------------------------- traits
test('the security trait adds a SECURITY.md requirement on top of the class', () => {
const f = checkRequiredFiles({ present: ['README.md', 'LICENSE'], klass: 'standalone', traits: ['security'] }, REGISTER);
assert.equal(f.some((x) => x.code === 'FILE-MISSING' && x.msg.includes('SECURITY.md')), true);
});
test('a repo without the security trait owes no SECURITY.md', () => {
const f = checkRequiredFiles({ present: ['README.md', 'LICENSE'], klass: 'standalone' }, REGISTER);
assert.equal(f.some((x) => x.msg.includes('SECURITY.md')), false);
});
test('the security trait requires limitations to be stated', () => {
const f = checkHeadings({ readme: '# x\n## Install\n## Non-goals\n', klass: 'standalone', traits: ['security'] }, REGISTER);
assert.equal(f.some((x) => x.code === 'HEADING-MISSING' && x.msg.includes('Known limitations')), true);
});
// Same fix as the ROADMAP test above: was checking the test's own fixture
// against itself. Reading the live register makes it a real guard again.
test('CONTRIBUTING and CODE_OF_CONDUCT are required by no class — the maintainer works alone', () => {
const live = loadRegister();
for (const klass of Object.keys(live.classes)) {
const req = live.classes[klass].required_files;
assert.equal(req.includes('CONTRIBUTING.md'), false);
assert.equal(req.includes('CODE_OF_CONDUCT.md'), false);
assert.equal(req.includes('MAINTAINERS.md'), false);
}
});
// ---------------------------------------------- link check: the noise sources
// All three found by running against llm-security, which produced ~30 false
// positives on the first pass. A check that is wrong this often teaches people
// to ignore it, which is worse than not having it.
test('a regex inside an inline code span is not a markdown link', () => {
// `["']([A-Za-z0-9\-._]{16,64})["']` is `[...](...)` to a naive scanner.
const line = '- **Regex:** `(?i)\\bapi[_\\-]?key\\s*[:=]\\s*["\']([A-Za-z0-9\\-._]{16,64})["\']`';
const f = checkInternalLinks({ files: { 'k.md': line }, present: ['k.md'] });
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
});
test('fenced code blocks are not scanned for links', () => {
const text = ['```bash', 'curl [x](not-a-real-file.md)', '```'].join('\n');
const f = checkInternalLinks({ files: { 'r.md': text }, present: ['r.md'] });
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
});
test('any URI scheme is left alone, not just http', () => {
const text = '[a](file:///abs/path.html) [b](ftp://x/y) [c](vscode://z)';
const f = checkInternalLinks({ files: { 'r.md': text }, present: ['r.md'] });
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
});
test('a path that escapes the repo is unresolvable, not broken', () => {
// Legitimately common: a plugin README pointing up at its marketplace.
// The gate sees one repo, so it cannot judge — and must not pretend to.
const f = checkInternalLinks({ files: { 'README.md': '[d](../../README.md)' }, present: ['README.md'] });
assert.equal(f.some((x) => x.code === 'LINK-OUTSIDE-REPO' && x.level === 'SKIP'), true);
assert.equal(f.some((x) => x.level === 'ERROR'), false);
});
test('a genuinely missing sibling file is still an ERROR', () => {
const f = checkInternalLinks({ files: { 'README.md': '[e](docs/gone.md)' }, present: ['README.md'] });
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING' && x.level === 'ERROR'), true);
});
// Reported by portfolio-optimiser-claude (coord 20260803T194933Z): `present` is
// the set of tracked FILES, so a directory link never has a member to match,
// even when every file under it is tracked. `have` never contained a directory
// to begin with — the fix derives one from `present`, it does not loosen it.
test('a link to a directory that is genuinely tracked resolves', () => {
const f = checkInternalLinks({
files: { 'README.md': 'See [dir](runs/s10/) and [file](runs/s10/a.json).' },
present: ['README.md', 'runs/s10/a.json'],
});
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), false);
});
test('a link to a directory with no tracked files under it is still an ERROR', () => {
const f = checkInternalLinks({
files: { 'README.md': '[ghost](docs/ghost/)' },
present: ['README.md'],
});
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING' && x.level === 'ERROR'), true);
});
test('a required heading present at the wrong level says so', () => {
// llm-security has `### Install` nested under `## Quick Start`. The contract
// wants it at level 2 — the finding should name that, not just "missing".
const f = checkHeadings({ readme: '# x\n## Quick Start\n### Install\n## Non-goals\n## Changelog\n', klass: 'plugin' }, REGISTER);
const hit = f.find((x) => x.code === 'HEADING-LEVEL');
assert.ok(hit, 'expected a HEADING-LEVEL finding');
assert.match(hit.msg, /### Install/);
});
test('a document ABOUT placeholders does not trip the placeholder detector', () => {
// llm-security documents the very strings it scans for. Inside code spans and
// fenced blocks they are subject matter, not unfinished template text.
const text = [
'Skip if the matched value contains: `your-project-name`, `FIXME`, `changeme`.',
'',
'```bash',
'cd <your-fork>',
'```',
].join('\n');
const f = checkBoilerplate({ files: { 'k.md': text } });
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
});
test('placeholder text in ordinary prose is still caught', () => {
const f = checkBoilerplate({ files: { 'README.md': 'Install your-project-name to begin.' } });
assert.equal(f.some((x) => x.code === 'BOILERPLATE'), true);
});
// --------------------------------- relative links resolve against their file
test('resolveRelative joins against the containing file, not the repo root', () => {
assert.equal(resolveRelative('examples/demo/README.md', 'expected.md'), 'examples/demo/expected.md');
assert.equal(resolveRelative('README.md', 'docs/design.md'), 'docs/design.md');
assert.equal(resolveRelative('docs/a/b.md', '../c.md'), 'docs/c.md');
assert.equal(resolveRelative('docs/a.md', './b.md'), 'docs/b.md');
});
test('resolveRelative returns null when the path escapes the repo', () => {
assert.equal(resolveRelative('README.md', '../../README.md'), null);
assert.equal(resolveRelative('docs/a.md', '../../../x.md'), null);
assert.equal(resolveRelative('README.md', '/etc/passwd'), null);
});
test('a link from a nested README to its sibling resolves', () => {
// Found against llm-security: both files existed, and the gate called them
// missing because it compared a file-relative target to repo-root paths.
const f = checkInternalLinks({
files: { 'examples/demo/README.md': 'See [findings](expected-findings.md).' },
present: ['examples/demo/README.md', 'examples/demo/expected-findings.md'],
});
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
});
test('a nested link to a genuinely absent sibling is still reported', () => {
const f = checkInternalLinks({
files: { 'examples/demo/README.md': 'See [gone](gone.md).' },
present: ['examples/demo/README.md'],
});
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), true);
});
// ------------------------------------------- link level follows the reader
// Measured: 30 of 43 LINK-INTERNAL-MISSING across the org sat in `shared/`,
// `docs/plan/` and `.claude/` — path-traversal test fixtures with deliberately
// invalid targets, internal session plans, agent working files. Only 12 were in
// a README. Every one of those 30 was an ERROR, which is how a gate gets
// switched off. Who the reader is decides what is required: the repo root is
// the shop window, everything below it is internal.
test('a dead link in a root document blocks a stranger — ERROR', () => {
const f = checkInternalLinks({ files: { 'README.md': '[e](docs/gone.md)' }, present: ['README.md'] });
const hit = f.find((x) => x.code === 'LINK-INTERNAL-MISSING');
assert.equal(hit.level, 'ERROR');
assert.equal(hit.bucket, 'broken');
});
test('a dead link below the root is a WARN, not an ERROR', () => {
// `shared/examples/nav-golden-escape/` in portfolio-optimiser links at
// `../../../../etc/passwd` ON PURPOSE — it is the fixture for a path-traversal
// test. Nineteen ERRORs against that repo were almost all this.
const f = checkInternalLinks({
files: { 'docs/plan/session.md': '[x](../gone.md)' },
present: ['docs/plan/session.md'],
});
const hit = f.find((x) => x.code === 'LINK-INTERNAL-MISSING');
assert.equal(hit.level, 'WARN');
assert.equal(hit.bucket, 'broken');
});
test('only the level moves — an internal dead link is never silently dropped', () => {
const f = checkInternalLinks({
files: { 'docs/a.md': '[x](gone.md)' },
present: ['docs/a.md'],
});
assert.equal(f.filter((x) => x.code === 'LINK-INTERNAL-MISSING').length, 1);
assert.match(f.find((x) => x.code === 'LINK-INTERNAL-MISSING').msg, /docs\/a\.md:1/);
});
test('WARN-only links must not also report that every link resolves', () => {
// The OK line said "every resolvable relative link resolves" whenever there
// was no ERROR. Degrading below-root links to WARN would have made that line
// appear beside its own counter-evidence.
const f = checkInternalLinks({
files: { 'docs/a.md': '[x](gone.md)' },
present: ['docs/a.md'],
});
assert.equal(f.some((x) => x.code === 'LINKS-INTERNAL'), false);
});
// ------------------------------------ fixture paths are presumed intentional
// `shared/examples/nav-golden-escape/bundle/index.md` in portfolio-optimiser
// deliberately escapes with `../../../../etc/passwd` — the deep `..` pops the
// whole base path and lands on `etc/passwd`, a path that is not `null` (still
// inside the repo by the resolver's arithmetic) and not tracked, so it read as
// a genuine WARN. It is the fixture doing its job, not a broken link. Third
// tool in the org to hit this same pattern — the check was the thing at fault.
test('a dead link inside a *golden* fixture path is SKIP with its own code, not WARN', () => {
const f = checkInternalLinks({
files: { 'shared/examples/nav-golden-escape/bundle/index.md': '[x](../../../../etc/passwd)' },
present: ['shared/examples/nav-golden-escape/bundle/index.md'],
});
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), false);
const hit = f.find((x) => x.code === 'LINK-INTERNAL-FIXTURE');
assert.equal(hit.level, 'SKIP');
assert.equal(hit.bucket, undefined);
});
test('a dead link inside a tests/ or fixtures/ directory is SKIP, not judged', () => {
const f = checkInternalLinks({
files: { 'tests/fixtures/plan.md': '[x](gone.md)' },
present: ['tests/fixtures/plan.md'],
});
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), false);
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-FIXTURE' && x.level === 'SKIP'), true);
});
test('a fixture-path dead link is never silently dropped — SKIP still names file and line', () => {
const f = checkInternalLinks({
files: { 'tests/plan.md': '[x](gone.md)' },
present: ['tests/plan.md'],
});
const hit = f.find((x) => x.code === 'LINK-INTERNAL-FIXTURE');
assert.match(hit.msg, /tests\/plan\.md:1/);
});
test('SKIP-classified fixture links do not suppress the every-link-resolves OK line', () => {
// LINK-OUTSIDE-REPO already sits outside the LINK-INTERNAL-MISSING check that
// gates the OK line; LINK-INTERNAL-FIXTURE follows the same precedent.
const f = checkInternalLinks({
files: { 'tests/plan.md': '[x](gone.md)' },
present: ['tests/plan.md'],
});
assert.equal(f.some((x) => x.code === 'LINKS-INTERNAL'), true);
});
test('a directory only substring-matching "test" or "fixtures" is not treated as a fixture path', () => {
// Exact segment match only for the literal names — "testing/" or
// "fixturesque/" are real directories, not the fixture convention. Only
// *golden* is a deliberate substring glob.
const f = checkInternalLinks({
files: { 'testing/plan.md': '[x](gone.md)' },
present: ['testing/plan.md'],
});
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-FIXTURE'), false);
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING' && x.level === 'WARN'), true);
});
test('a root-level dead link is unaffected by the fixture heuristic', () => {
const f = checkInternalLinks({ files: { 'README.md': '[e](docs/gone.md)' }, present: ['README.md'] });
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-FIXTURE'), false);
assert.equal(f.find((x) => x.code === 'LINK-INTERNAL-MISSING').level, 'ERROR');
});
// ------------------------------------------------ the repo name is the remote
// `catalog/` is the working directory of the repo named `ktg-plugin-marketplace`.
// Deriving the name from the directory basename left it REPO-UNREGISTERED and
// ran zero checks against the one repo the whole catalog rule depends on.
test('the repo name comes from the remote, not the directory', () => {
assert.equal(
parseRepoNameFromRemote('ssh://git@git.fromaitochitta.com/open/ktg-plugin-marketplace.git'),
'ktg-plugin-marketplace',
);
assert.equal(
parseRepoNameFromRemote('https://git.fromaitochitta.com/open/repo-standard.git'),
'repo-standard',
);
});
test('remote parsing handles the scp form and a missing .git suffix', () => {
// The forge UI hands out the scp form from its clone button.
assert.equal(parseRepoNameFromRemote('git@git.fromaitochitta.com:open/okr.git'), 'okr');
assert.equal(parseRepoNameFromRemote('https://git.fromaitochitta.com/open/okr'), 'okr');
assert.equal(parseRepoNameFromRemote('https://git.fromaitochitta.com/open/okr/'), 'okr');
});
test('an absent or unparseable remote yields null, so the caller can fall back', () => {
assert.equal(parseRepoNameFromRemote(''), null);
assert.equal(parseRepoNameFromRemote(null), null);
assert.equal(parseRepoNameFromRemote(' '), null);
assert.equal(parseRepoNameFromRemote('https://git.fromaitochitta.com/'), null);
});
// ------------------------------- stripCode must not swallow nested list items
test('a link inside a nested list item is still scanned', () => {
// An indented line is only a code block when a blank line precedes it.
// Treating every 4-space indent as code made nested bullets invisible to both
// the link and boilerplate checks — a silent false pass, which is worse than
// the noise it was meant to remove.
const text = ['- top level', ' - nested with [a link](gone.md)'].join('\n');
const f = checkInternalLinks({ files: { 'README.md': text }, present: ['README.md'] });
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), true);
});
test('a genuine indented code block is still skipped', () => {
const text = ['Example:', '', ' curl [x](not-real.md)'].join('\n');
const f = checkInternalLinks({ files: { 'README.md': text }, present: ['README.md'] });
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
});
// ------------------------------------------ badge honesty beyond shields.io
test('a self-hosted badge asserting a run is caught too', () => {
const f = checkBadges({ readme: '![Tests](https://example.com/static/tests-34-green.svg)' });
assert.equal(f.some((x) => x.code === 'BADGE-STATIC-CLAIM'), true);
});
// ------------------------------------------------------------ install truth
test('the marketplace URL in the install block must be the real one', () => {
const readme = [
'## Install',
'claude plugin marketplace add https://git.fromaitochitta.com/open/WRONG.git',
`claude plugin install repo-standard@${MKT.name}`,
].join('\n');
const f = checkInstallBlock({ readme, name: 'repo-standard', klass: 'plugin' }, REGISTER);
assert.equal(f.some((x) => x.code === 'INSTALL-URL-MISMATCH' && x.bucket === 'broken'), true);
});
test('a plugin absent from the catalog has an install line that cannot work', () => {
// Syntax is not truth. The brief's first control asks whether the command
// works for a stranger, not whether it is well-formed.
const f = checkInstallTruth({ name: 'ghost', klass: 'plugin', catalogNames: ['repo-mailbox', 'llm-security'] });
assert.equal(f.some((x) => x.code === 'INSTALL-NOT-IN-CATALOG' && x.bucket === 'broken'), true);
});
test('a plugin present in the catalog passes install truth', () => {
const f = checkInstallTruth({ name: 'repo-mailbox', klass: 'plugin', catalogNames: ['repo-mailbox'] });
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
});
test('an unreachable catalog is SKIP, never a pass', () => {
const f = checkInstallTruth({ name: 'repo-mailbox', klass: 'plugin', catalogNames: null });
assert.equal(f[0].level, 'SKIP');
});
test('non-plugin classes are not measured against the catalog', () => {
const f = checkInstallTruth({ name: 'portfolio-optimiser', klass: 'standalone', catalogNames: null });
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
});
test('an indented code block stays code past its first line', () => {
// The blank-line rule fixed nested lists but broke multi-line indented blocks:
// only line 1 followed a blank line, so lines 2+ leaked back into scanning.
// Caught by the gate on this plugin's own SKILL.md, which shows a README
// template containing a link that does not exist relative to that file.
const text = [
'Template:',
'',
' # <name>',
' ## Changelog',
' See [CHANGELOG.md](CHANGELOG.md).',
].join('\n');
const f = checkInternalLinks({ files: { 'skills/x/SKILL.md': text }, present: ['skills/x/SKILL.md'] });
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
});
test('an indented block ends at the first unindented line', () => {
const text = ['Template:', '', ' code here', '', 'Back to prose with [a link](gone.md).'].join('\n');
const f = checkInternalLinks({ files: { 'README.md': text }, present: ['README.md'] });
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), true);
});
// -------------------------------------------------------- fetchWithRetry
// fetchOrgListing/fetchCatalogNames are I/O shell, exercised live by the CLI —
// but the retry decision is pure once fetchImpl and sleep are injected, so it
// gets the same unit coverage as the classifiers. Measured 2026-08-04: 13
// script invocations in ~1s tripped an anonymous 429 with no retry at all;
// this is the fix, not a sweep tool (that belongs in org-ops).
function fakeResponse(status, headers = {}) {
return {
status,
ok: status >= 200 && status < 300,
headers: { get: (k) => headers[k.toLowerCase()] ?? null },
};
}
test('a 429 is retried once and a subsequent 200 is returned', async () => {
const calls = [];
const responses = [fakeResponse(429), fakeResponse(200)];
const fetchImpl = async () => responses.shift();
const sleep = async (ms) => calls.push(ms);
const res = await fetchWithRetry('https://x', {}, { fetchImpl, sleep });
assert.equal(res.status, 200);
assert.equal(calls.length, 1);
});
test('Retry-After (seconds) sets the wait, not the exponential default', async () => {
const calls = [];
const responses = [fakeResponse(429, { 'retry-after': '2' }), fakeResponse(200)];
const fetchImpl = async () => responses.shift();
const sleep = async (ms) => calls.push(ms);
await fetchWithRetry('https://x', {}, { fetchImpl, sleep });
assert.equal(calls[0], 2000);
});
test('no Retry-After header falls back to exponential backoff from baseDelayMs', async () => {
const calls = [];
const responses = [fakeResponse(429), fakeResponse(429), fakeResponse(200)];
const fetchImpl = async () => responses.shift();
const sleep = async (ms) => calls.push(ms);
await fetchWithRetry('https://x', {}, { fetchImpl, sleep, baseDelayMs: 1000 });
assert.deepEqual(calls, [1000, 2000]);
});
test('exponential backoff is capped, so a long retry budget does not wait minutes between attempts', async () => {
// Measured 2026-08-04 against the live forge: 20 parallel requests from one
// IP produced 429 with NO Retry-After header at all (nginx never sends one
// here) — the exponential fallback is the only path that ever runs in
// practice. Recovery was gradual, not a fixed-duration ban: a burst that
// size took up to ~15s to fully drain, and a 20s manual pause cleared it.
// Uncapped doubling would reach 32s on a single attempt; capping at 8s and
// extending the retry budget covers the measured recovery window without
// one attempt blocking for excessive time.
const calls = [];
const responses = [
fakeResponse(429),
fakeResponse(429),
fakeResponse(429),
fakeResponse(429),
fakeResponse(429),
fakeResponse(200),
];
const fetchImpl = async () => responses.shift();
const sleep = async (ms) => calls.push(ms);
await fetchWithRetry('https://x', {}, { fetchImpl, sleep, baseDelayMs: 1000, maxDelayMs: 8000, retries: 5 });
// Uncapped, the 5th delay would be 1000 * 2**4 = 16000.
assert.deepEqual(calls, [1000, 2000, 4000, 8000, 8000]);
});
test('retries are bounded — a persistent 429 returns the 429, not an infinite loop', async () => {
let calls = 0;
const fetchImpl = async () => fakeResponse(429);
const sleep = async () => {
calls += 1;
};
const res = await fetchWithRetry('https://x', {}, { fetchImpl, sleep, retries: 2, baseDelayMs: 1 });
assert.equal(res.status, 429);
assert.equal(calls, 2);
});
test('a non-429 response returns immediately — no retry, no sleep', async () => {
let fetchCalls = 0;
let sleepCalls = 0;
const fetchImpl = async () => {
fetchCalls += 1;
return fakeResponse(200);
};
const sleep = async () => {
sleepCalls += 1;
};
const res = await fetchWithRetry('https://x', {}, { fetchImpl, sleep });
assert.equal(res.status, 200);
assert.equal(fetchCalls, 1);
assert.equal(sleepCalls, 0);
});
// --------------------------------------------------- engine version stamp
// Measured 2026-08-04: the /repo-standard skill resolved to a cached 0.1.1
// plugin root while 0.2.0 was installed and the catalog pinned it. Two repos
// independently proved it by running both engines against the same checkout:
// 0.2.0 gave 0 WARN, 0.1.1 resurrected the three false positives fixed in
// v0.1.2/v0.1.3 — and nothing in the output said which engine had run. The
// header carrying no version is what let a stale engine look like a pass.
test('headerLine names the engine version, not just the repo status', () => {
const line = headerLine({ name: 'voyage', klass: 'plugin', traits: [], status: 'OK' }, '0.2.1');
assert.match(line, /voyage/);
assert.match(line, /OK/);
assert.match(line, /0\.2\.1/);
});
test('headerLine includes traits when present, same as the unversioned line did', () => {
const line = headerLine({ name: 'llm-security', klass: 'plugin', traits: ['security'], status: 'WARN' }, '0.2.1');
assert.match(line, /\{security\}/);
});
test('withEngineVersion adds the version without disturbing existing fields', () => {
const result = { name: 'x', klass: 'plugin', status: 'OK', findings: [] };
const stamped = withEngineVersion(result, '0.2.1');
assert.equal(stamped.engineVersion, '0.2.1');
assert.equal(stamped.name, 'x');
assert.equal(stamped.status, 'OK');
assert.deepEqual(result, { name: 'x', klass: 'plugin', status: 'OK', findings: [] });
});
// ---------------------------------------------------- engine commit stamp
// Measured 2026-08-09 by org-ops: a sweep stamped 18 raw files `0.4.0`, and
// four of them carried findings from a check that only exists in 0.5.0. The
// stamp lied without being broken — `9eb210b` (the feature) and `63c75d8` (the
// version bump) are TWO commits, so the worktree held new code under an old
// number for a window. The version names a FILE; only the sha names the CODE,
// and the bump is by definition a different commit from the change it describes.
test('headerLine names the engine commit alongside the version', () => {
const line = headerLine({ name: 'voyage', klass: 'plugin', traits: [], status: 'OK' }, '0.5.0', 'a7276e6f0d1e2b3c');
assert.match(line, /0\.5\.0/);
assert.match(line, /a7276e6/);
});
test('headerLine falls back to the version alone when the commit is underivable', () => {
const line = headerLine({ name: 'voyage', klass: 'plugin', traits: [], status: 'OK' }, '0.5.0', null);
assert.match(line, /0\.5\.0/);
assert.doesNotMatch(line, /null|undefined/);
});
// Letting `status` mean judgement fixes "clean repos look skipped"; printing an
// unqualified OK beside a check that never ran would trade it for "skipped
// checks look clean", which is the worse direction. The one line org-ops reads
// has to carry both axes.
test('headerLine qualifies a pass that had checks it could not run', () => {
const line = headerLine({ name: 'okr', klass: 'plugin', traits: [], status: 'OK', notChecked: 1 }, '0.7.0');
assert.match(line, /OK/);
assert.match(line, /1 not checked/);
});
test('headerLine says nothing about coverage when every check ran', () => {
const line = headerLine({ name: 'okr', klass: 'plugin', traits: [], status: 'OK', notChecked: 0 }, '0.7.0');
assert.doesNotMatch(line, /not checked/);
});
// An older result object has no `notChecked` at all. Absent is not zero and not
// a crash — it prints the pre-0.7.0 line rather than the word "undefined".
test('headerLine tolerates a result from before the coverage axis existed', () => {
const line = headerLine({ name: 'voyage', klass: 'plugin', traits: [], status: 'OK' }, '0.7.0');
assert.doesNotMatch(line, /not checked|undefined|NaN/);
});
test('withEngineVersion stamps the commit next to the version', () => {
const stamped = withEngineVersion({ name: 'x', status: 'OK', findings: [] }, '0.5.0', 'a7276e6f0d1e2b3c');
assert.equal(stamped.engineVersion, '0.5.0');
assert.equal(stamped.engineCommit, 'a7276e6f0d1e2b3c');
});
test('engineCommit is present-and-null when underivable, never absent', () => {
// An ABSENT key means an older engine; an explicit null means this engine ran
// and could not derive the sha. A consumer sorting raw files by stamp needs
// those two to be different, which is the whole point of the field.
const stamped = withEngineVersion({ name: 'x', status: 'OK', findings: [] }, '0.5.0', null);
assert.equal(Object.hasOwn(stamped, 'engineCommit'), true);
assert.equal(stamped.engineCommit, null);
});
test('readEngineCommit returns null outside a git checkout instead of throwing', () => {
// Derived from the engine's own checkout with no network call. A consumer of
// a tarball or a vendored copy has no HEAD, and that is not a crash.
assert.equal(readEngineCommit('/nonexistent-path-for-repo-standard-test'), null);
});
// ------------------------------------------------- the two kinds of SKIP
// v0.7.0 took SKIP out of the severity lattice and gave coverage its own axis,
// which fixed "clean repos look skipped". It left one thing merged that org-ops
// had already named (20260809T124015Z, observation 2): a SKIP that means
// "deliberately not judged" is not the same fact as a SKIP that means "could
// not run". Measured over the 05 census, 35 of 39 skips are the first kind —
// so `portfolio-optimiser — OK · 11 not checked` reads as eleven unread checks
// when all eleven are links the gate declines to judge on purpose.
//
// The tag is set at the EMISSION site, never looked up from the code. Codes are
// not the carrier of that meaning: `VERSION-TAG` is emitted at SKIP when there
// are no tags and at ERROR when the tag is missing, so a code→kind map cannot
// express the split without re-deriving the reason it already knew.
test('groupSkips separates the deliberate from the un-runnable', () => {
const g = groupSkips([
{ level: 'SKIP', skip: 'byDesign', code: 'LINK-OUTSIDE-REPO' },
{ level: 'SKIP', skip: 'notRun', code: 'INSTALL-TRUTH' },
{ level: 'SKIP', skip: 'byDesign', code: 'LINK-INTERNAL-FIXTURE' },
]);
assert.deepEqual(g.byDesign.map((f) => f.code), ['LINK-OUTSIDE-REPO', 'LINK-INTERNAL-FIXTURE']);
assert.deepEqual(g.notRun.map((f) => f.code), ['INSTALL-TRUTH']);
});
test('groupSkips ignores judged findings — this axis exists only under SKIP', () => {
const g = groupSkips([
{ level: 'ERROR', code: 'VERSION-TAG', bucket: 'broken' },
{ level: 'OK', code: 'VERSION' },
]);
assert.deepEqual(g, { byDesign: [], notRun: [] });
});
// An untagged SKIP can only come from an older engine or a site the source scan
// below has not caught yet. It lands in `notRun`, which is the loud side: a
// skip of unknown kind must not inherit "deliberate, nothing to see".
test('an untagged SKIP counts as un-runnable, not as deliberate', () => {
assert.deepEqual(skipsOf([{ level: 'SKIP', code: 'MYSTERY' }]), { byDesign: 0, notRun: 1 });
});
test('skipsOf counts both kinds beside the status, same as bucketsOf does', () => {
const counts = skipsOf([
{ level: 'SKIP', skip: 'byDesign', code: 'LINK-OUTSIDE-REPO' },
{ level: 'SKIP', skip: 'byDesign', code: 'LINK-INTERNAL-FIXTURE' },
{ level: 'SKIP', skip: 'notRun', code: 'VERSION-TAG' },
{ level: 'OK', code: 'VERSION' },
]);
assert.deepEqual(counts, { byDesign: 2, notRun: 1 });
});
// The source scan is the test that survives the next quarter. A corpus test
// only sees the sites it manages to trigger; the eleventh emission site added
// later is invisible to it and would silently take the reader's default.
test('every SKIP the engine can emit is tagged where it is emitted', () => {
const src = readFileSync(new URL('./repo-standard-check.mjs', import.meta.url), 'utf8');
const untagged = [];
const re = /level: 'SKIP'/g;
for (const m of src.matchAll(re)) {
// Convention, so this stays checkable: the tag sits between `level` and
// `code` in the object literal. Anything else reads as untagged.
const tail = src.slice(m.index, src.indexOf('code:', m.index));
if (!/skip: '(byDesign|notRun)'/.test(tail)) {
untagged.push(src.slice(0, m.index).split('\n').length);
}
}
assert.deepEqual(untagged, [], `untagged SKIP emission at line(s) ${untagged.join(', ')}`);
});
// Which side each site lands on. The discriminator is not "how bad" but "can
// this ever become a verdict, and does anyone have an action?".
test('a link out of the repo is deliberate — the gate sees one repo by design', () => {
const f = checkInternalLinks({ files: { 'README.md': '[x](../sibling/file.md)' }, present: ['README.md'] });
const hit = f.find((x) => x.code === 'LINK-OUTSIDE-REPO');
assert.equal(hit.skip, 'byDesign');
});
test('a fixture-path dead link is deliberate — presumed intentional, nothing to fix', () => {
const f = checkInternalLinks({ files: { 'tests/plan.md': '[x](gone.md)' }, present: ['tests/plan.md'] });
assert.equal(f.find((x) => x.code === 'LINK-INTERNAL-FIXTURE').skip, 'byDesign');
});
test('no tags yet is un-runnable — a release resolves it', () => {
const f = checkVersionConsistency({ pluginVersion: '0.7.1', readmeBadge: null, changelogTop: null, tags: [] });
assert.equal(f.find((x) => x.code === 'VERSION-TAG').skip, 'notRun');
});
// Was `SKIP`/`notRun` until 0.9.0, deferred once on the stated ground that
// re-levelling "moves a repo's status". Measured false: an added `OK` cannot
// worsen the worst *judged* finding, and all three repos that emit this already
// read `OK`. It is the `checkReadmeLanguage` shape — the check ran, saw
// everything, and found no subject.
test('no version claimed anywhere is nothing to judge, not a skip', () => {
const f = checkVersionConsistency({ pluginVersion: null, readmeBadge: null, changelogTop: null, tags: [] });
const hit = f.find((x) => x.code === 'VERSION-NONE');
assert.equal(hit.level, 'OK');
assert.equal(hit.skip, undefined);
});
// The rename is the point, not cosmetics: `-UNAVAILABLE` is this engine's
// naming for a `notRun` skip (`DESC-UNAVAILABLE`, `INSTALL-TRUTH`). One
// OK-level `-UNAVAILABLE` would mislead exactly the reader this change serves.
test('the un-runnable version codes keep the -UNAVAILABLE naming', () => {
const src = readFileSync(new URL('./repo-standard-check.mjs', import.meta.url), 'utf8');
assert.ok(!src.includes('VERSION-UNAVAILABLE'), 'VERSION-UNAVAILABLE was renamed to VERSION-NONE');
});
test('an unreachable catalog is un-runnable — a second run resolves it', () => {
const f = checkInstallTruth({ name: 'repo-mailbox', klass: 'plugin', catalogNames: null });
assert.equal(f.find((x) => x.code === 'INSTALL-TRUTH').skip, 'notRun');
});
test('an unregistered repo is un-runnable — registering it resolves it', () => {
const r = classifyRepo({ name: 'stranger', files: {}, present: [], description: null }, REGISTER);
assert.equal(r.findings[0].skip, 'notRun');
});
test('classifyRepo carries both skip counts, and notChecked stays the unchanged total', () => {
// `notChecked` must keep being a NUMBER. A consumer doing `notChecked > 0` on
// an object gets a silent false, which would kill the header qualifier the
// same way the old status hid clean repos.
const r = classifyRepo({ name: 'stranger', files: {}, present: [], description: null }, REGISTER);
assert.equal(typeof r.notChecked, 'number');
assert.deepEqual(r.skips, { byDesign: 0, notRun: 1 });
assert.equal(r.notChecked, r.skips.byDesign + r.skips.notRun);
});
// The header line is the whole defect. If the split lands only in `--json`, the
// symptom org-ops actually reads survives untouched. Operator decision
// 2026-08-09: the line names only what someone has an action on. The deliberate
// ones are not silenced — they keep their own sub-heading in the body, where
// "exposure, not silence" actually lives.
test('headerLine names the checks that could not run', () => {
const line = headerLine(
{ name: 'x', klass: 'plugin', traits: [], status: 'ERROR', notChecked: 11, skips: { byDesign: 9, notRun: 2 } },
'0.8.0',
);
assert.match(line, /2 not run/);
});
test('headerLine says nothing when every skip was a deliberate one', () => {
const line = headerLine(
{ name: 'portfolio-optimiser', klass: 'tool', traits: [], status: 'OK', notChecked: 11, skips: { byDesign: 11, notRun: 0 } },
'0.8.0',
);
assert.doesNotMatch(line, /not run|not checked|11/);
assert.match(line, /OK/);
});
test('headerLine still prints the pre-0.8.0 line for a result with no skip split', () => {
// Absent `skips` means an older result object, exactly as an absent
// `notChecked` means one older still. Neither is zero.
const line = headerLine({ name: 'okr', klass: 'plugin', traits: [], status: 'OK', notChecked: 1 }, '0.8.0');
assert.match(line, /1 not checked/);
});