Measured this build against a documentation brief for public repos. The five original checks covered roughly one of its ten sections, so this adds what a single repo can answer on its own. New: required README headings per class (Non-goals is the cheapest trust-builder there is), in-repo version consistency across manifest / badge / CHANGELOG / tag, badge honesty, boilerplate, licence-claim, and relative links. Findings now carry a BUCKET beside the level - broken / missing / weakening - and output is grouped by it, because that is the order the work gets done in. Traits are a second axis beside class: class is structural and readable off the catalog, a trait says what the code does. `security` attaches SECURITY.md and a Known limitations section. The two names carrying it are proposed, not measured - that list is the operator's. Solo-maintained settles a category: CONTRIBUTING, CODE_OF_CONDUCT and MAINTAINERS are required by no class. Consumer-facing documents are untouched by that; SECURITY.md exists for the stranger who finds a hole. Three bugs found by running against llm-security, not by reading: - ~30 link findings, all noise. Regexes inside code spans are `[...](...)` to a naive scanner. Strip code first. - `file:` and other schemes were treated as repo-relative paths. - Relative links were resolved against the repo root instead of the file they sit in, calling two files missing that sat next to the README linking them. Same fix applied to the boilerplate check: a document ABOUT placeholder detection was tripping the placeholder detector. Also removed this repo's own static tests badge. There is no CI - the forge has zero Actions runners registered - so it could never become real, and it is the exact anti-pattern the gate now flags. 67 tests. Against llm-security every remaining finding is real and matches the census's independent hand-measurement. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WYJ3FHLtVgzFXMZ6UF598h
696 lines
31 KiB
JavaScript
696 lines
31 KiB
JavaScript
// Tests for the repo-standard gate.
|
|
//
|
|
// The pure classifiers are the unit under test — the I/O shell (inspectRepo/runGate)
|
|
// is exercised against a live checkout by the CLI, not here.
|
|
//
|
|
// The link-check fixtures are the six measured false positives from the census.
|
|
// They are the reason this gate has three outcomes instead of a boolean.
|
|
import { test } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import {
|
|
countCodepoints,
|
|
normalizeRepoRef,
|
|
extractOpenRefs,
|
|
classifyRef,
|
|
checkLinks,
|
|
checkDescription,
|
|
checkFirstScreen,
|
|
checkInstallBlock,
|
|
checkRequiredFiles,
|
|
checkVersionConsistency,
|
|
checkHeadings,
|
|
checkBadges,
|
|
checkBoilerplate,
|
|
checkLicenseClaim,
|
|
checkInternalLinks,
|
|
resolveRelative,
|
|
classifyRepo,
|
|
levelOf,
|
|
} from './repo-standard-check.mjs';
|
|
|
|
const REGISTER = {
|
|
org: 'open',
|
|
marketplace: {
|
|
name: 'ktg-plugin-marketplace',
|
|
url: 'https://git.fromaitochitta.com/open/ktg-plugin-marketplace.git',
|
|
},
|
|
repos: {
|
|
'llm-security': 'plugin',
|
|
'repo-mailbox': 'plugin',
|
|
'repo-standard': 'plugin',
|
|
'ktg-plugin-marketplace': 'catalog',
|
|
'playground-design-system': 'shared-asset',
|
|
'.profile': 'org-profile',
|
|
'llm-ingestion-pipeline-security': 'standalone',
|
|
},
|
|
non_repos: {
|
|
coord: 'Retired repo name, deliberately still alive in prose: the CLI, the mailbox root and CLAUDE_COORD_DIR kept it — they are the transport protocol, not the product.',
|
|
_broadcast: 'reserved engine namespace',
|
|
'llm-ingestion-guard': 'package name, not a repo',
|
|
'claude-code-llm-security': 'pre-split name of llm-security',
|
|
},
|
|
classes: {
|
|
plugin: {
|
|
required_files: ['README.md', 'LICENSE', 'CHANGELOG.md', '.claude-plugin/plugin.json'],
|
|
required_headings: ['## Install', '## Non-goals', '## Changelog'],
|
|
install: 'plugin',
|
|
},
|
|
catalog: {
|
|
required_files: ['README.md', 'LICENSE', 'GOVERNANCE.md', 'CONVENTIONS.md', '.claude-plugin/marketplace.json'],
|
|
required_headings: ['## Install', '## Non-goals'],
|
|
install: 'catalog',
|
|
},
|
|
'shared-asset': { required_files: ['README.md', 'LICENSE'], required_headings: ['## Non-goals'], install: 'vendor' },
|
|
'org-profile': { required_files: ['README.md'], required_headings: [], install: 'none' },
|
|
standalone: { required_files: ['README.md', 'LICENSE'], required_headings: ['## Install', '## Non-goals'], install: 'package' },
|
|
},
|
|
traits: {
|
|
'llm-security': ['security'],
|
|
'llm-ingestion-pipeline-security': ['security'],
|
|
},
|
|
trait_requirements: {
|
|
security: {
|
|
required_files: ['SECURITY.md'],
|
|
required_headings: ['## Known limitations'],
|
|
},
|
|
},
|
|
description_max_codepoints: 180,
|
|
};
|
|
|
|
// ---------------------------------------------------------------- measurement
|
|
|
|
test('countCodepoints measures codepoints, not bytes and not UTF-16 units', () => {
|
|
// The em-dash exposes only the byte layer: 3 bytes, 1 codepoint, 1 UTF-16 unit.
|
|
assert.equal(countCodepoints('a—b'), 3);
|
|
assert.equal(Buffer.byteLength('a—b', 'utf8'), 5);
|
|
// 👉 is astral: 1 codepoint but 2 UTF-16 units. This is the layer the em-dash hides.
|
|
assert.equal(countCodepoints('👉'), 1);
|
|
assert.equal('👉'.length, 2);
|
|
});
|
|
|
|
// ------------------------------------------------------------- ref extraction
|
|
|
|
test('normalizeRepoRef strips a .git suffix and a trailing slash', () => {
|
|
// ~20 "dead" names collapsed to 3 real ones once .git was normalised.
|
|
assert.equal(normalizeRepoRef('llm-security.git'), 'llm-security');
|
|
assert.equal(normalizeRepoRef('llm-security/'), 'llm-security');
|
|
assert.equal(normalizeRepoRef('llm-security'), 'llm-security');
|
|
});
|
|
|
|
test('extractOpenRefs finds names in URL position only', () => {
|
|
const text = [
|
|
'clone https://git.fromaitochitta.com/open/llm-security.git today',
|
|
'see https://git.fromaitochitta.com/open/repo-mailbox/src/branch/main/README.md',
|
|
].join('\n');
|
|
const names = extractOpenRefs(text).map((r) => r.name);
|
|
assert.deepEqual(names, ['llm-security', 'repo-mailbox']);
|
|
});
|
|
|
|
test('extractOpenRefs ignores path position, prose and bare directory names', () => {
|
|
// False positives #4, #5, #6 — the text is correct and will STAY correct.
|
|
const text = [
|
|
'the mailbox root is ~/.claude/coord/_broadcast/inbox/',
|
|
'coord is the transport protocol, not the product',
|
|
'the catalog lives in ktg-plugin-marketplace/catalog',
|
|
'the package llm-ingestion-guard is published from that repo',
|
|
].join('\n');
|
|
assert.deepEqual(extractOpenRefs(text), []);
|
|
});
|
|
|
|
test('extractOpenRefs handles the ssh scp-style form', () => {
|
|
const refs = extractOpenRefs('git@git.fromaitochitta.com:open/llm-security.git');
|
|
assert.deepEqual(refs.map((r) => r.name), ['llm-security']);
|
|
});
|
|
|
|
test('extractOpenRefs reports the 1-indexed line of each hit', () => {
|
|
const text = 'line one\nline two\nhttps://git.fromaitochitta.com/open/llm-security';
|
|
assert.equal(extractOpenRefs(text)[0].line, 3);
|
|
});
|
|
|
|
// ---------------------------------------------- three outcomes, not a boolean
|
|
|
|
test('classifyRef separates repo, non-repo and unknown', () => {
|
|
assert.equal(classifyRef('llm-security', REGISTER), 'repo');
|
|
assert.equal(classifyRef('.profile', REGISTER), 'repo');
|
|
assert.equal(classifyRef('coord', REGISTER), 'non-repo');
|
|
assert.equal(classifyRef('nonesuch', REGISTER), 'unknown');
|
|
});
|
|
|
|
test('a URL-position ref to a known non-repo is a DISTINCT outcome from no match', () => {
|
|
// The specification requirement: "no match" and "match on something that is
|
|
// not a repo" must never share an outcome, or the loss goes silent.
|
|
const bad = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/nonesuch' } }, REGISTER);
|
|
const odd = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/coord' } }, REGISTER);
|
|
|
|
assert.equal(bad[0].level, 'ERROR');
|
|
assert.equal(bad[0].code, 'LINK-DEAD');
|
|
|
|
assert.equal(odd[0].level, 'WARN');
|
|
assert.equal(odd[0].code, 'LINK-NON-REPO');
|
|
assert.notEqual(bad[0].code, odd[0].code);
|
|
// The reason travels with the finding, so the reader is not sent measuring again.
|
|
assert.match(odd[0].msg, /transport protocol/);
|
|
});
|
|
|
|
test('a dead ref names its successor when the register knows one', () => {
|
|
const f = checkLinks(
|
|
{ files: { 'README.md': 'https://git.fromaitochitta.com/open/claude-code-llm-security' } },
|
|
REGISTER,
|
|
);
|
|
assert.equal(f[0].level, 'WARN');
|
|
assert.match(f[0].msg, /pre-split name/);
|
|
});
|
|
|
|
test('the .git suffix does not manufacture a dead reference', () => {
|
|
const f = checkLinks(
|
|
{ files: { 'README.md': 'https://git.fromaitochitta.com/open/llm-security.git' } },
|
|
REGISTER,
|
|
);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('the hidden .profile resolves — the enumerator sees what a glob misses', () => {
|
|
const f = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/.profile' } }, REGISTER);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
// ------------------------------------------------------------- description
|
|
|
|
test('description length is bounded, measured in codepoints', () => {
|
|
assert.equal(checkDescription('a fine description', REGISTER)[0].level, 'OK');
|
|
assert.equal(checkDescription('', REGISTER)[0].level, 'ERROR');
|
|
assert.equal(checkDescription('x'.repeat(181), REGISTER)[0].level, 'ERROR');
|
|
assert.equal(checkDescription('x'.repeat(180), REGISTER)[0].level, 'OK');
|
|
});
|
|
|
|
test('an unavailable description is SKIP, never a pass', () => {
|
|
// Offline is not compliance. A check that could not run says so.
|
|
const f = checkDescription(null, REGISTER);
|
|
assert.equal(f[0].level, 'SKIP');
|
|
});
|
|
|
|
// ------------------------------------------------------------- first screen
|
|
|
|
test('README line 1 must be the H1, and the description line must match the forge', () => {
|
|
const readme = '# repo-mailbox\nA local mailbox for coordination.\n';
|
|
assert.equal(
|
|
checkFirstScreen({ readme, name: 'repo-mailbox', description: 'A local mailbox for coordination.' })
|
|
.every((f) => f.level === 'OK'),
|
|
true,
|
|
);
|
|
});
|
|
|
|
test('a README whose opening line diverges from the description is an ERROR', () => {
|
|
const readme = '# repo-mailbox\nSomething else entirely.\n';
|
|
const f = checkFirstScreen({ readme, name: 'repo-mailbox', description: 'A local mailbox for coordination.' });
|
|
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'README-DESC'), true);
|
|
});
|
|
|
|
test('first-screen description match is SKIP when the forge text is unavailable', () => {
|
|
const f = checkFirstScreen({ readme: '# x\nbody\n', name: 'x', description: null });
|
|
assert.equal(f.some((x) => x.code === 'README-DESC' && x.level === 'SKIP'), true);
|
|
});
|
|
|
|
test('a missing H1 is an ERROR', () => {
|
|
const f = checkFirstScreen({ readme: 'no heading here\n', name: 'x', description: null });
|
|
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'README-H1'), true);
|
|
});
|
|
|
|
test('an H1 that differs from the repo name is a WARN, not a failure', () => {
|
|
// `# OKR for Public Sector` is a naming choice, not a defect: the thread that
|
|
// must hold is description == catalog == opening line, and the H1 is none of
|
|
// those three. Surface it; let the operator decide.
|
|
const f = checkFirstScreen({ readme: '# OKR for Public Sector\nbody\n', name: 'okr', description: null });
|
|
assert.equal(f.some((x) => x.level === 'WARN' && x.code === 'README-H1'), true);
|
|
assert.equal(f.some((x) => x.level === 'ERROR'), false);
|
|
});
|
|
|
|
test('a differing H1 does not stop the description check from running', () => {
|
|
const f = checkFirstScreen({ readme: '# Nice Title\nthe description\n', name: 'x', description: 'the description' });
|
|
assert.equal(f.some((x) => x.code === 'README-DESC' && x.level === 'OK'), true);
|
|
});
|
|
|
|
// ------------------------------------------------------------ install block
|
|
|
|
const MKT = REGISTER.marketplace;
|
|
|
|
test('plugin install needs BOTH lines: marketplace add and a CLI install command', () => {
|
|
const readme = [
|
|
'## Install',
|
|
'```',
|
|
`claude plugin marketplace add ${MKT.url}`,
|
|
`claude plugin install repo-mailbox@${MKT.name}`,
|
|
'```',
|
|
].join('\n');
|
|
const f = checkInstallBlock({ readme, name: 'repo-mailbox', klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('the slash form counts as the CLI command', () => {
|
|
const readme = [
|
|
'## Install',
|
|
`claude plugin marketplace add ${MKT.url}`,
|
|
`/plugin install claude-design@${MKT.name}`,
|
|
].join('\n');
|
|
const f = checkInstallBlock({ readme, name: 'claude-design', klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('enabledPlugins JSON is an ALLOWED ADDITION, never a replacement for the CLI command', () => {
|
|
// This is the corrected defect A: 7 of 11 plugin READMEs stop after
|
|
// `marketplace add`. The JSON form works and stands in 10 of 11 — what is
|
|
// missing is a CLI command, so the contract requires the command and permits
|
|
// the JSON alongside it.
|
|
const jsonOnly = [
|
|
'## Install',
|
|
`claude plugin marketplace add ${MKT.url}`,
|
|
'Or enable directly in `~/.claude/settings.json`:',
|
|
`"enabledPlugins": { "llm-security@${MKT.name}": true }`,
|
|
].join('\n');
|
|
const f = checkInstallBlock({ readme: jsonOnly, name: 'llm-security', klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'INSTALL-NO-CLI'), true);
|
|
|
|
const both = jsonOnly + `\nclaude plugin install llm-security@${MKT.name}\n`;
|
|
const g = checkInstallBlock({ readme: both, name: 'llm-security', klass: 'plugin' }, REGISTER);
|
|
assert.equal(g.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('a missing marketplace add is its own finding — okr and claude-design are opposite halves', () => {
|
|
const noAdd = ['## Install', `/plugin install claude-design@${MKT.name}`].join('\n');
|
|
const f = checkInstallBlock({ readme: noAdd, name: 'claude-design', klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'INSTALL-NO-MARKETPLACE'), true);
|
|
|
|
// okr: neither line. Both findings fire — a rule saying "both lines" must not
|
|
// hit this repo blind.
|
|
const neither = ['## Install', `"enabledPlugins": { "okr@${MKT.name}": true }`].join('\n');
|
|
const g = checkInstallBlock({ readme: neither, name: 'okr', klass: 'plugin' }, REGISTER);
|
|
assert.equal(g.some((x) => x.code === 'INSTALL-NO-MARKETPLACE'), true);
|
|
assert.equal(g.some((x) => x.code === 'INSTALL-NO-CLI'), true);
|
|
});
|
|
|
|
test('the install target must name THIS repo, not another plugin', () => {
|
|
const readme = [
|
|
'## Install',
|
|
`claude plugin marketplace add ${MKT.url}`,
|
|
`claude plugin install some-other-plugin@${MKT.name}`,
|
|
].join('\n');
|
|
const f = checkInstallBlock({ readme, name: 'repo-standard', klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'INSTALL-NO-CLI'), true);
|
|
});
|
|
|
|
test('ssh in the marketplace add line is an ERROR — marketplace add rejects it', () => {
|
|
// Measured end-to-end: `marketplace add ssh://...` → "Invalid git URL".
|
|
// The forge UI's clone button hands you exactly that URL.
|
|
const readme = [
|
|
'## Install',
|
|
'claude plugin marketplace add ssh://git@git.fromaitochitta.com/open/ktg-plugin-marketplace.git',
|
|
`claude plugin install repo-standard@${MKT.name}`,
|
|
].join('\n');
|
|
const f = checkInstallBlock({ readme, name: 'repo-standard', klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'INSTALL-SSH'), true);
|
|
});
|
|
|
|
test('the install block is parametric — a different marketplace passes on its own values', () => {
|
|
// wiki-advise lives on the private ktg/ namespace and is distributed via
|
|
// `ktg-privat`. A skill that hardcodes the public marketplace produces an
|
|
// install line that does not work there — and being public itself, this skill
|
|
// cannot carry private marketplace names.
|
|
const priv = {
|
|
...REGISTER,
|
|
marketplace: { name: 'ktg-privat', url: 'https://git.fromaitochitta.com/ktg/ktg-privat.git' },
|
|
};
|
|
const readme = [
|
|
'## Install',
|
|
'claude plugin marketplace add https://git.fromaitochitta.com/ktg/ktg-privat.git',
|
|
'claude plugin install wiki-advise@ktg-privat',
|
|
].join('\n');
|
|
const f = checkInstallBlock({ readme, name: 'wiki-advise', klass: 'plugin' }, priv);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('the catalog needs only marketplace add — it IS the marketplace', () => {
|
|
const readme = `## Install\nclaude plugin marketplace add ${MKT.url}`;
|
|
const f = checkInstallBlock({ readme, name: 'ktg-plugin-marketplace', klass: 'catalog' }, REGISTER);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('a shared asset is vendored, not installed — a plugin install line is wrong there', () => {
|
|
const asPlugin = `## Install\nclaude plugin install playground-design-system@${MKT.name}`;
|
|
const f = checkInstallBlock(
|
|
{ readme: asPlugin, name: 'playground-design-system', klass: 'shared-asset' },
|
|
REGISTER,
|
|
);
|
|
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'INSTALL-WRONG-FORM'), true);
|
|
});
|
|
|
|
test('.profile needs no install section at all', () => {
|
|
const f = checkInstallBlock({ readme: '# .profile\nOrg profile.\n', name: '.profile', klass: 'org-profile' }, REGISTER);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
// ----------------------------------------------------------- required files
|
|
|
|
test('required files are per class, not flat across the org', () => {
|
|
const ok = checkRequiredFiles({ present: ['README.md'], klass: 'org-profile' }, REGISTER);
|
|
assert.equal(ok.filter((f) => f.level === 'ERROR').length, 0);
|
|
|
|
const missing = checkRequiredFiles({ present: ['README.md'], klass: 'plugin' }, REGISTER);
|
|
assert.equal(missing.some((f) => f.code === 'FILE-MISSING' && f.msg.includes('LICENSE')), true);
|
|
});
|
|
|
|
test('no class requires a ROADMAP — it is 0/18 and belongs to a later step', () => {
|
|
for (const klass of Object.keys(REGISTER.classes)) {
|
|
assert.equal(REGISTER.classes[klass].required_files.includes('ROADMAP.md'), false);
|
|
}
|
|
});
|
|
|
|
// -------------------------------------------------------------- aggregation
|
|
|
|
test('levelOf ranks ERROR above WARN above SKIP above OK', () => {
|
|
assert.equal(levelOf([{ level: 'OK' }, { level: 'WARN' }, { level: 'ERROR' }]), 'ERROR');
|
|
assert.equal(levelOf([{ level: 'OK' }, { level: 'WARN' }]), 'WARN');
|
|
assert.equal(levelOf([{ level: 'OK' }, { level: 'SKIP' }]), 'SKIP');
|
|
assert.equal(levelOf([{ level: 'OK' }]), 'OK');
|
|
assert.equal(levelOf([]), 'OK');
|
|
});
|
|
|
|
test('an unregistered repo is SKIP, not a pass — the gate refuses to guess a class', () => {
|
|
const r = classifyRepo({ name: 'stranger', files: {}, present: [], description: null }, REGISTER);
|
|
assert.equal(r.status, 'SKIP');
|
|
});
|
|
|
|
test('a fully compliant plugin repo classifies OK', () => {
|
|
const readme = [
|
|
'# repo-mailbox',
|
|
'A local mailbox for coordination.',
|
|
'',
|
|
'Body text.',
|
|
'',
|
|
'',
|
|
'',
|
|
'## Install',
|
|
`claude plugin marketplace add ${MKT.url}`,
|
|
`claude plugin install repo-mailbox@${MKT.name}`,
|
|
'',
|
|
'## Non-goals',
|
|
'It is not a state store.',
|
|
'',
|
|
'## Changelog',
|
|
'See [CHANGELOG.md](CHANGELOG.md).',
|
|
].join('\n');
|
|
const r = classifyRepo(
|
|
{
|
|
name: 'repo-mailbox',
|
|
files: { 'README.md': readme },
|
|
present: ['README.md', 'LICENSE', 'CHANGELOG.md', '.claude-plugin/plugin.json'],
|
|
description: 'A local mailbox for coordination.',
|
|
pluginVersion: '0.7.0',
|
|
readmeBadge: '0.7.0',
|
|
changelogTop: '0.7.0',
|
|
tags: ['v0.7.0'],
|
|
},
|
|
REGISTER,
|
|
);
|
|
assert.equal(r.status, 'OK');
|
|
assert.deepEqual(r.buckets, { broken: 0, missing: 0, weakening: 0 });
|
|
});
|
|
|
|
test('the security trait travels from the register into the verdict', () => {
|
|
const r = classifyRepo(
|
|
{
|
|
name: 'llm-security',
|
|
files: { 'README.md': '# llm-security\ndesc\n## Install\n## Non-goals\n## Changelog\n' },
|
|
present: ['README.md', 'LICENSE', 'CHANGELOG.md', '.claude-plugin/plugin.json'],
|
|
description: 'desc',
|
|
},
|
|
REGISTER,
|
|
);
|
|
assert.deepEqual(r.traits, ['security']);
|
|
assert.equal(r.findings.some((f) => f.code === 'FILE-MISSING' && f.msg.includes('SECURITY.md')), true);
|
|
assert.equal(r.findings.some((f) => f.code === 'HEADING-MISSING' && f.msg.includes('Known limitations')), true);
|
|
});
|
|
|
|
// =========================================================================
|
|
// The brief's reporting contract: findings carry a BUCKET alongside a level.
|
|
// "broken now" (strangers are blocked or misled), "missing" (an expected
|
|
// artefact is absent), "weakening" (present, but reads as amateur). The two
|
|
// axes are independent: a weakening finding can still be an ERROR.
|
|
// =========================================================================
|
|
|
|
test('every ERROR and WARN finding carries a bucket', () => {
|
|
const readme = ['## Install', `"enabledPlugins": { "okr@${MKT.name}": true }`].join('\n');
|
|
const all = [
|
|
...checkInstallBlock({ readme, name: 'okr', klass: 'plugin' }, REGISTER),
|
|
...checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/nonesuch' } }, REGISTER),
|
|
...checkRequiredFiles({ present: ['README.md'], klass: 'plugin' }, REGISTER),
|
|
...checkDescription('', REGISTER),
|
|
];
|
|
const graded = all.filter((f) => f.level === 'ERROR' || f.level === 'WARN');
|
|
assert.ok(graded.length > 0);
|
|
for (const f of graded) {
|
|
assert.ok(['broken', 'missing', 'weakening'].includes(f.bucket), `${f.code} has bucket ${f.bucket}`);
|
|
}
|
|
});
|
|
|
|
test('an unusable install path is `broken`, an absent file is `missing`', () => {
|
|
const readme = ['## Install', 'nothing useful here'].join('\n');
|
|
const inst = checkInstallBlock({ readme, name: 'okr', klass: 'plugin' }, REGISTER);
|
|
assert.equal(inst.find((f) => f.code === 'INSTALL-NO-CLI').bucket, 'broken');
|
|
|
|
const files = checkRequiredFiles({ present: ['README.md'], klass: 'plugin' }, REGISTER);
|
|
assert.equal(files.find((f) => f.code === 'FILE-MISSING').bucket, 'missing');
|
|
});
|
|
|
|
// ------------------------------------------------------- version consistency
|
|
|
|
test('version consistency: manifest, README badge and CHANGELOG must agree', () => {
|
|
const ok = checkVersionConsistency({ pluginVersion: '0.1.0', readmeBadge: '0.1.0', changelogTop: '0.1.0', tags: ['v0.1.0'] });
|
|
assert.equal(ok.filter((f) => f.level === 'ERROR').length, 0);
|
|
|
|
const drift = checkVersionConsistency({ pluginVersion: '0.1.0', readmeBadge: '0.0.9', changelogTop: '0.1.0', tags: ['v0.1.0'] });
|
|
assert.equal(drift.some((f) => f.level === 'ERROR' && f.code === 'VERSION-BADGE'), true);
|
|
|
|
const stale = checkVersionConsistency({ pluginVersion: '0.2.0', readmeBadge: '0.2.0', changelogTop: '0.1.0', tags: ['v0.1.0', 'v0.2.0'] });
|
|
assert.equal(stale.some((f) => f.level === 'ERROR' && f.code === 'VERSION-CHANGELOG'), true);
|
|
});
|
|
|
|
test('an untagged repo SKIPs the tag comparison rather than failing it', () => {
|
|
// Nothing has been released yet. That is a state, not a defect — and "SKIP is
|
|
// never a pass" means it must say so rather than quietly succeed.
|
|
const f = checkVersionConsistency({ pluginVersion: '0.1.0', readmeBadge: '0.1.0', changelogTop: '0.1.0', tags: [] });
|
|
assert.equal(f.some((x) => x.code === 'VERSION-TAG' && x.level === 'SKIP'), true);
|
|
assert.equal(f.some((x) => x.level === 'ERROR'), false);
|
|
});
|
|
|
|
test('a released version with no matching tag is an ERROR', () => {
|
|
const f = checkVersionConsistency({ pluginVersion: '0.3.1', readmeBadge: '0.3.1', changelogTop: '0.3.1', tags: ['v0.1.0'] });
|
|
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'VERSION-TAG'), true);
|
|
});
|
|
|
|
// -------------------------------------------------------- required headings
|
|
|
|
test('required headings are per class — Non-goals is required, not optional', () => {
|
|
const full = '# x\n## Install\n## Non-goals\n## Changelog\n';
|
|
assert.equal(checkHeadings({ readme: full, klass: 'plugin' }, REGISTER).filter((f) => f.level === 'ERROR').length, 0);
|
|
|
|
const noNonGoals = '# x\n## Install\n## Changelog\n';
|
|
const f = checkHeadings({ readme: noNonGoals, klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'HEADING-MISSING' && x.msg.includes('Non-goals')), true);
|
|
assert.equal(f.find((x) => x.code === 'HEADING-MISSING').bucket, 'missing');
|
|
});
|
|
|
|
test('org-profile requires no headings at all', () => {
|
|
const f = checkHeadings({ readme: '# .profile\nprofile\n', klass: 'org-profile' }, REGISTER);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
// ------------------------------------------------------------ badge honesty
|
|
|
|
test('a static badge asserting test or build status is a finding', () => {
|
|
// "tests: 642 passing" as a static image is a claim dressed as evidence.
|
|
const f = checkBadges({ readme: '' });
|
|
assert.equal(f.some((x) => x.code === 'BADGE-STATIC-CLAIM'), true);
|
|
assert.equal(f.find((x) => x.code === 'BADGE-STATIC-CLAIM').bucket, 'weakening');
|
|
});
|
|
|
|
test('static version, licence and platform badges are fine — they assert no run', () => {
|
|
const readme = [
|
|
'',
|
|
'',
|
|
'',
|
|
].join('\n');
|
|
assert.equal(checkBadges({ readme }).filter((f) => f.level !== 'OK').length, 0);
|
|
});
|
|
|
|
test('a build badge that links to a real run is fine', () => {
|
|
const readme = '[](https://forge.example/repo/actions)';
|
|
assert.equal(checkBadges({ readme }).filter((f) => f.level !== 'OK').length, 0);
|
|
});
|
|
|
|
// -------------------------------------------------------------- boilerplate
|
|
|
|
test('unfinished template text is a finding', () => {
|
|
const f = checkBoilerplate({ files: { 'README.md': 'Install your-project-name today' } });
|
|
assert.equal(f.some((x) => x.code === 'BOILERPLATE'), true);
|
|
|
|
const g = checkBoilerplate({ files: { 'CODE_OF_CONDUCT.md': 'Report to [INSERT EMAIL ADDRESS]' } });
|
|
assert.equal(g.some((x) => x.code === 'BOILERPLATE'), true);
|
|
});
|
|
|
|
test('ordinary prose is not boilerplate', () => {
|
|
const f = checkBoilerplate({ files: { 'README.md': 'This project solves a real problem.' } });
|
|
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
|
|
});
|
|
|
|
// --------------------------------------------------- licence claim vs. file
|
|
|
|
test('a README that cites a LICENSE the repo does not have is broken', () => {
|
|
const f = checkLicenseClaim({ readme: 'Released under the [MIT licence](LICENSE).', present: ['README.md'] });
|
|
assert.equal(f.some((x) => x.code === 'LICENSE-CLAIMED-ABSENT' && x.bucket === 'broken'), true);
|
|
});
|
|
|
|
test('a cited LICENSE that exists passes', () => {
|
|
const f = checkLicenseClaim({ readme: 'See [LICENSE](LICENSE).', present: ['README.md', 'LICENSE'] });
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
// ------------------------------------------------------------ internal links
|
|
|
|
test('a relative link to a file that does not exist is a finding', () => {
|
|
const f = checkInternalLinks(
|
|
{ files: { 'README.md': 'See [the design](docs/design.md).' }, present: ['README.md'] },
|
|
);
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), true);
|
|
});
|
|
|
|
test('relative links that resolve, anchors and external URLs are left alone', () => {
|
|
const f = checkInternalLinks({
|
|
files: { 'README.md': '[a](docs/design.md) [b](#section) [c](https://example.com) [d](mailto:x@y.z)' },
|
|
present: ['README.md', 'docs/design.md'],
|
|
});
|
|
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
|
|
});
|
|
|
|
// ----------------------------------------------------------------- traits
|
|
|
|
test('the security trait adds a SECURITY.md requirement on top of the class', () => {
|
|
const f = checkRequiredFiles({ present: ['README.md', 'LICENSE'], klass: 'standalone', traits: ['security'] }, REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'FILE-MISSING' && x.msg.includes('SECURITY.md')), true);
|
|
});
|
|
|
|
test('a repo without the security trait owes no SECURITY.md', () => {
|
|
const f = checkRequiredFiles({ present: ['README.md', 'LICENSE'], klass: 'standalone' }, REGISTER);
|
|
assert.equal(f.some((x) => x.msg.includes('SECURITY.md')), false);
|
|
});
|
|
|
|
test('the security trait requires limitations to be stated', () => {
|
|
const f = checkHeadings({ readme: '# x\n## Install\n## Non-goals\n', klass: 'standalone', traits: ['security'] }, REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'HEADING-MISSING' && x.msg.includes('Known limitations')), true);
|
|
});
|
|
|
|
test('CONTRIBUTING and CODE_OF_CONDUCT are required by no class — the maintainer works alone', () => {
|
|
for (const klass of Object.keys(REGISTER.classes)) {
|
|
const req = REGISTER.classes[klass].required_files;
|
|
assert.equal(req.includes('CONTRIBUTING.md'), false);
|
|
assert.equal(req.includes('CODE_OF_CONDUCT.md'), false);
|
|
assert.equal(req.includes('MAINTAINERS.md'), false);
|
|
}
|
|
});
|
|
|
|
// ---------------------------------------------- link check: the noise sources
|
|
// All three found by running against llm-security, which produced ~30 false
|
|
// positives on the first pass. A check that is wrong this often teaches people
|
|
// to ignore it, which is worse than not having it.
|
|
|
|
test('a regex inside an inline code span is not a markdown link', () => {
|
|
// `["']([A-Za-z0-9\-._]{16,64})["']` is `[...](...)` to a naive scanner.
|
|
const line = '- **Regex:** `(?i)\\bapi[_\\-]?key\\s*[:=]\\s*["\']([A-Za-z0-9\\-._]{16,64})["\']`';
|
|
const f = checkInternalLinks({ files: { 'k.md': line }, present: ['k.md'] });
|
|
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
|
|
});
|
|
|
|
test('fenced code blocks are not scanned for links', () => {
|
|
const text = ['```bash', 'curl [x](not-a-real-file.md)', '```'].join('\n');
|
|
const f = checkInternalLinks({ files: { 'r.md': text }, present: ['r.md'] });
|
|
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
|
|
});
|
|
|
|
test('any URI scheme is left alone, not just http', () => {
|
|
const text = '[a](file:///abs/path.html) [b](ftp://x/y) [c](vscode://z)';
|
|
const f = checkInternalLinks({ files: { 'r.md': text }, present: ['r.md'] });
|
|
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
|
|
});
|
|
|
|
test('a path that escapes the repo is unresolvable, not broken', () => {
|
|
// Legitimately common: a plugin README pointing up at its marketplace.
|
|
// The gate sees one repo, so it cannot judge — and must not pretend to.
|
|
const f = checkInternalLinks({ files: { 'README.md': '[d](../../README.md)' }, present: ['README.md'] });
|
|
assert.equal(f.some((x) => x.code === 'LINK-OUTSIDE-REPO' && x.level === 'SKIP'), true);
|
|
assert.equal(f.some((x) => x.level === 'ERROR'), false);
|
|
});
|
|
|
|
test('a genuinely missing sibling file is still an ERROR', () => {
|
|
const f = checkInternalLinks({ files: { 'README.md': '[e](docs/gone.md)' }, present: ['README.md'] });
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING' && x.level === 'ERROR'), true);
|
|
});
|
|
|
|
test('a required heading present at the wrong level says so', () => {
|
|
// llm-security has `### Install` nested under `## Quick Start`. The contract
|
|
// wants it at level 2 — the finding should name that, not just "missing".
|
|
const f = checkHeadings({ readme: '# x\n## Quick Start\n### Install\n## Non-goals\n## Changelog\n', klass: 'plugin' }, REGISTER);
|
|
const hit = f.find((x) => x.code === 'HEADING-LEVEL');
|
|
assert.ok(hit, 'expected a HEADING-LEVEL finding');
|
|
assert.match(hit.msg, /### Install/);
|
|
});
|
|
|
|
test('a document ABOUT placeholders does not trip the placeholder detector', () => {
|
|
// llm-security documents the very strings it scans for. Inside code spans and
|
|
// fenced blocks they are subject matter, not unfinished template text.
|
|
const text = [
|
|
'Skip if the matched value contains: `your-project-name`, `FIXME`, `changeme`.',
|
|
'',
|
|
'```bash',
|
|
'cd <your-fork>',
|
|
'```',
|
|
].join('\n');
|
|
const f = checkBoilerplate({ files: { 'k.md': text } });
|
|
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
|
|
});
|
|
|
|
test('placeholder text in ordinary prose is still caught', () => {
|
|
const f = checkBoilerplate({ files: { 'README.md': 'Install your-project-name to begin.' } });
|
|
assert.equal(f.some((x) => x.code === 'BOILERPLATE'), true);
|
|
});
|
|
|
|
// --------------------------------- relative links resolve against their file
|
|
|
|
test('resolveRelative joins against the containing file, not the repo root', () => {
|
|
assert.equal(resolveRelative('examples/demo/README.md', 'expected.md'), 'examples/demo/expected.md');
|
|
assert.equal(resolveRelative('README.md', 'docs/design.md'), 'docs/design.md');
|
|
assert.equal(resolveRelative('docs/a/b.md', '../c.md'), 'docs/c.md');
|
|
assert.equal(resolveRelative('docs/a.md', './b.md'), 'docs/b.md');
|
|
});
|
|
|
|
test('resolveRelative returns null when the path escapes the repo', () => {
|
|
assert.equal(resolveRelative('README.md', '../../README.md'), null);
|
|
assert.equal(resolveRelative('docs/a.md', '../../../x.md'), null);
|
|
assert.equal(resolveRelative('README.md', '/etc/passwd'), null);
|
|
});
|
|
|
|
test('a link from a nested README to its sibling resolves', () => {
|
|
// Found against llm-security: both files existed, and the gate called them
|
|
// missing because it compared a file-relative target to repo-root paths.
|
|
const f = checkInternalLinks({
|
|
files: { 'examples/demo/README.md': 'See [findings](expected-findings.md).' },
|
|
present: ['examples/demo/README.md', 'examples/demo/expected-findings.md'],
|
|
});
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('a nested link to a genuinely absent sibling is still an ERROR', () => {
|
|
const f = checkInternalLinks({
|
|
files: { 'examples/demo/README.md': 'See [gone](gone.md).' },
|
|
present: ['examples/demo/README.md'],
|
|
});
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), true);
|
|
});
|