Kjell Tore Guttormsen
5884a64e54
Three changes at the register/engine boundary, all agreed with org-ops after census 05 and all about a check missing a place to record a legitimate exception. `titles`: an optional per-repo README title. Set, the H1 matches it and is OK; unset, the WARN stands as before. A human title was already this engine's stated position and rds-v1's prescription, but a decided YES had nowhere to live, so the same 6 WARNs were reported three censuses running and would have been reported forever. Five registered, each H1 read from the repo rather than copied from the census; `ai-psychosis` deliberately left out so the one repo where a reader cannot connect title to name stands alone. Measured across 21 local clones: 6 WARN before, 1 after, nothing else moved. `readme_desc_match: false` on the org-profile class: for an ordinary repo the README opening and the forge description describe the same subject and equality is right; for this class they do not — the README is the org's landing page, the forge text describes the repo. The equality is what does not apply, not either text. Class data, not a hardcoded name, and the exemption is RECORDED as an OK naming its reason, not dropped. `.profile` went ERROR to 0 ERROR / 0 WARN; the same README under a plugin class is still an ERROR. `engineCommit`: the version names a file, only the sha names the code. A sweep stamped 18 files 0.4.0 while four carried a 0.5.0-only finding — feature and version bump are two commits, so the stamp lied without being broken. Present-and-null when underivable, never absent: an absent key means an older engine, null means this one ran without a HEAD to read. 135 to 147 tests. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NELsvPY5gnJjN3esdhWYWC
1505 lines
71 KiB
JavaScript
1505 lines
71 KiB
JavaScript
// Tests for the repo-standard gate.
|
|
//
|
|
// The pure classifiers are the unit under test — the I/O shell (inspectRepo/runGate)
|
|
// is exercised against a live checkout by the CLI, not here.
|
|
//
|
|
// The link-check fixtures are the six measured false positives from the census.
|
|
// They are the reason this gate has three outcomes instead of a boolean.
|
|
import { test } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import {
|
|
countCodepoints,
|
|
normalizeRepoRef,
|
|
extractOpenRefs,
|
|
classifyRef,
|
|
checkLinks,
|
|
checkDescription,
|
|
checkFirstScreen,
|
|
checkInstallBlock,
|
|
checkRequiredFiles,
|
|
checkVersionConsistency,
|
|
checkHeadings,
|
|
checkBadges,
|
|
checkReadmeLanguage,
|
|
checkBoilerplate,
|
|
checkLicenseClaim,
|
|
checkInternalLinks,
|
|
resolveRelative,
|
|
checkInstallTruth,
|
|
classifyRepo,
|
|
levelOf,
|
|
parseRepoNameFromRemote,
|
|
extractChangelogTop,
|
|
fetchWithRetry,
|
|
headerLine,
|
|
withEngineVersion,
|
|
readEngineCommit,
|
|
loadRegister,
|
|
} from './repo-standard-check.mjs';
|
|
|
|
const REGISTER = {
|
|
org: 'open',
|
|
marketplace: {
|
|
name: 'ktg-plugin-marketplace',
|
|
url: 'https://git.fromaitochitta.com/open/ktg-plugin-marketplace.git',
|
|
},
|
|
repos: {
|
|
'llm-security': 'plugin',
|
|
'repo-mailbox': 'plugin',
|
|
'repo-standard': 'plugin',
|
|
'ktg-plugin-marketplace': 'catalog',
|
|
'playground-design-system': 'shared-asset',
|
|
'.profile': 'org-profile',
|
|
'llm-ingestion-pipeline-security': 'standalone',
|
|
},
|
|
non_repos: {
|
|
coord: 'Retired repo name, deliberately still alive in prose: the CLI, the mailbox root and CLAUDE_COORD_DIR kept it — they are the transport protocol, not the product.',
|
|
_broadcast: 'reserved engine namespace',
|
|
'llm-ingestion-guard': 'package name, not a repo',
|
|
'claude-code-llm-security': 'pre-split name of llm-security',
|
|
},
|
|
classes: {
|
|
plugin: {
|
|
required_files: ['README.md', 'LICENSE', 'CHANGELOG.md', '.claude-plugin/plugin.json'],
|
|
required_headings: ['## Install', '## Non-goals', '## Changelog'],
|
|
install: 'plugin',
|
|
},
|
|
catalog: {
|
|
required_files: ['README.md', 'LICENSE', 'GOVERNANCE.md', 'CONVENTIONS.md', '.claude-plugin/marketplace.json'],
|
|
required_headings: ['## Install', '## Non-goals'],
|
|
install: 'catalog',
|
|
},
|
|
'shared-asset': { required_files: ['README.md', 'LICENSE'], required_headings: ['## Non-goals'], install: 'vendor' },
|
|
'org-profile': { required_files: ['README.md'], required_headings: [], install: 'none', readme_desc_match: false },
|
|
standalone: { required_files: ['README.md', 'LICENSE'], required_headings: ['## Install', '## Non-goals'], install: 'package' },
|
|
},
|
|
titles: {
|
|
'llm-security': 'LLM Security Plugin for Claude Code',
|
|
},
|
|
traits: {
|
|
'llm-security': ['security'],
|
|
'llm-ingestion-pipeline-security': ['security'],
|
|
},
|
|
trait_requirements: {
|
|
security: {
|
|
required_files: ['SECURITY.md'],
|
|
required_headings: ['## Known limitations'],
|
|
},
|
|
},
|
|
description_max_codepoints: 180,
|
|
};
|
|
|
|
// ---------------------------------------------------------------- measurement
|
|
|
|
test('countCodepoints measures codepoints, not bytes and not UTF-16 units', () => {
|
|
// The em-dash exposes only the byte layer: 3 bytes, 1 codepoint, 1 UTF-16 unit.
|
|
assert.equal(countCodepoints('a—b'), 3);
|
|
assert.equal(Buffer.byteLength('a—b', 'utf8'), 5);
|
|
// 👉 is astral: 1 codepoint but 2 UTF-16 units. This is the layer the em-dash hides.
|
|
assert.equal(countCodepoints('👉'), 1);
|
|
assert.equal('👉'.length, 2);
|
|
});
|
|
|
|
// ------------------------------------------------------------- ref extraction
|
|
|
|
test('normalizeRepoRef strips a .git suffix and a trailing slash', () => {
|
|
// ~20 "dead" names collapsed to 3 real ones once .git was normalised.
|
|
assert.equal(normalizeRepoRef('llm-security.git'), 'llm-security');
|
|
assert.equal(normalizeRepoRef('llm-security/'), 'llm-security');
|
|
assert.equal(normalizeRepoRef('llm-security'), 'llm-security');
|
|
});
|
|
|
|
test('extractOpenRefs finds names in URL position only', () => {
|
|
const text = [
|
|
'clone https://git.fromaitochitta.com/open/llm-security.git today',
|
|
'see https://git.fromaitochitta.com/open/repo-mailbox/src/branch/main/README.md',
|
|
].join('\n');
|
|
const names = extractOpenRefs(text).map((r) => r.name);
|
|
assert.deepEqual(names, ['llm-security', 'repo-mailbox']);
|
|
});
|
|
|
|
test('extractOpenRefs ignores path position, prose and bare directory names', () => {
|
|
// False positives #4, #5, #6 — the text is correct and will STAY correct.
|
|
const text = [
|
|
'the mailbox root is ~/.claude/coord/_broadcast/inbox/',
|
|
'coord is the transport protocol, not the product',
|
|
'the catalog lives in ktg-plugin-marketplace/catalog',
|
|
'the package llm-ingestion-guard is published from that repo',
|
|
].join('\n');
|
|
assert.deepEqual(extractOpenRefs(text), []);
|
|
});
|
|
|
|
test('extractOpenRefs ignores an org segment inside an API endpoint path', () => {
|
|
// Measured false positive (catalog RUNBOOK.md:39, :114): the Forgejo API
|
|
// path /api/v1/orgs/open/repos has "open" as the org argument to the API,
|
|
// not a repo reference — "repos" is the literal API resource segment, and
|
|
// "open" is not the first path segment after the host.
|
|
const text = 'curl -X POST https://git.fromaitochitta.com/api/v1/orgs/open/repos';
|
|
assert.deepEqual(extractOpenRefs(text), []);
|
|
});
|
|
|
|
test('extractOpenRefs handles the ssh scp-style form', () => {
|
|
const refs = extractOpenRefs('git@git.fromaitochitta.com:open/llm-security.git');
|
|
assert.deepEqual(refs.map((r) => r.name), ['llm-security']);
|
|
});
|
|
|
|
test('extractOpenRefs finds a ref on a schemaless host — the scheme is not what makes it a URL', () => {
|
|
// Measured false negative (org-ops census 03, llm-security/V3-UPGRADE.md:343):
|
|
// a subtree instruction writes the host without a scheme. The name is still
|
|
// in URL position — `open` is the first segment after a dotted host — and it
|
|
// is still a name the register can resolve. Requiring `://` hid a WARN.
|
|
const refs = extractOpenRefs('- Subtree push to `git.fromaitochitta.com/open/claude-code-llm-security`');
|
|
assert.deepEqual(refs.map((r) => r.name), ['claude-code-llm-security']);
|
|
});
|
|
|
|
test('the schemaless form does not reopen the API-endpoint false positive', () => {
|
|
// The whole point of the host segment excluding `/` survives losing the
|
|
// scheme: `orgs` is the segment before `open`, so `open` is not first.
|
|
assert.deepEqual(extractOpenRefs('curl git.fromaitochitta.com/api/v1/orgs/open/repos'), []);
|
|
});
|
|
|
|
test('a dotted PATH segment is not a host — only a host puts a name in URL position', () => {
|
|
// The guard that keeps the schemaless branch from matching anywhere a dot
|
|
// happens to precede `open/`: a path segment is preceded by `/`, a host is not.
|
|
assert.deepEqual(extractOpenRefs('see https://git.fromaitochitta.com/docs/v1.2/open/notes'), []);
|
|
assert.deepEqual(extractOpenRefs('the fixture lives at test/nav.golden/open/index.md'), []);
|
|
});
|
|
|
|
test('a schemaless host is counted ONCE, not twice, when a scheme is present', () => {
|
|
// Two alternatives can match the same text; the leftmost consumes it.
|
|
const refs = extractOpenRefs('https://git.fromaitochitta.com/open/llm-security');
|
|
assert.deepEqual(refs.map((r) => r.name), ['llm-security']);
|
|
});
|
|
|
|
test('extractOpenRefs reports the 1-indexed line of each hit', () => {
|
|
const text = 'line one\nline two\nhttps://git.fromaitochitta.com/open/llm-security';
|
|
assert.equal(extractOpenRefs(text)[0].line, 3);
|
|
});
|
|
|
|
// ---------------------------------------------- three outcomes, not a boolean
|
|
|
|
test('classifyRef separates repo, non-repo and unknown', () => {
|
|
assert.equal(classifyRef('llm-security', REGISTER), 'repo');
|
|
assert.equal(classifyRef('.profile', REGISTER), 'repo');
|
|
assert.equal(classifyRef('coord', REGISTER), 'non-repo');
|
|
assert.equal(classifyRef('nonesuch', REGISTER), 'unknown');
|
|
});
|
|
|
|
test('a URL-position ref to a known non-repo is a DISTINCT outcome from no match', () => {
|
|
// The specification requirement: "no match" and "match on something that is
|
|
// not a repo" must never share an outcome, or the loss goes silent.
|
|
const bad = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/nonesuch' } }, REGISTER);
|
|
const odd = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/coord' } }, REGISTER);
|
|
|
|
assert.equal(bad[0].level, 'ERROR');
|
|
assert.equal(bad[0].code, 'LINK-DEAD');
|
|
|
|
assert.equal(odd[0].level, 'WARN');
|
|
assert.equal(odd[0].code, 'LINK-NON-REPO');
|
|
assert.notEqual(bad[0].code, odd[0].code);
|
|
// The reason travels with the finding, so the reader is not sent measuring again.
|
|
assert.match(odd[0].msg, /transport protocol/);
|
|
});
|
|
|
|
test('a dead ref names its successor when the register knows one', () => {
|
|
const f = checkLinks(
|
|
{ files: { 'README.md': 'https://git.fromaitochitta.com/open/claude-code-llm-security' } },
|
|
REGISTER,
|
|
);
|
|
assert.equal(f[0].level, 'WARN');
|
|
assert.match(f[0].msg, /pre-split name/);
|
|
});
|
|
|
|
test('the .git suffix does not manufacture a dead reference', () => {
|
|
const f = checkLinks(
|
|
{ files: { 'README.md': 'https://git.fromaitochitta.com/open/llm-security.git' } },
|
|
REGISTER,
|
|
);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('the hidden .profile resolves — the enumerator sees what a glob misses', () => {
|
|
const f = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/.profile' } }, REGISTER);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('checkLinks emits OK when every reference resolves — silence is not a pass', () => {
|
|
// Measured gap (org-ops census 03b): "no dead references" and "the check
|
|
// never ran" produced identical output, so a sweep across 19 repos could not
|
|
// tell 19 clean repos from 19 unread ones. The engine already applies "three
|
|
// outcomes, never two" to classifyRef; this applies it to its own result.
|
|
const f = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/llm-security' } }, REGISTER);
|
|
assert.equal(f.length, 1);
|
|
assert.equal(f[0].level, 'OK');
|
|
assert.equal(f[0].code, 'LINKS-OPEN-REFS');
|
|
// The count is the evidence: an OK that cannot say how many it checked is
|
|
// the same silence in a different colour.
|
|
assert.match(f[0].msg, /1 /);
|
|
});
|
|
|
|
test('a repo with no open/ references at all is still a ran check, not a blank', () => {
|
|
const f = checkLinks({ files: { 'README.md': '# title\n\nno references here' } }, REGISTER);
|
|
assert.equal(f.length, 1);
|
|
assert.equal(f[0].level, 'OK');
|
|
assert.match(f[0].msg, /no `open\/` references/);
|
|
});
|
|
|
|
test('checkLinks SKIPs when no files were enumerated — that is the "did not run" case', () => {
|
|
const f = checkLinks({ files: {} }, REGISTER);
|
|
assert.equal(f.length, 1);
|
|
assert.equal(f[0].level, 'SKIP');
|
|
assert.equal(f[0].code, 'LINKS-OPEN-REFS-UNAVAILABLE');
|
|
});
|
|
|
|
test('a link whose display text repeats its own URL is ONE reference, not two', () => {
|
|
// Introduced by the schemaless branch and measured before shipping it
|
|
// (llm-security/V3-ANNOUNCEMENT.md:124): `[host/open/x](https://host/open/x)`
|
|
// matches on both halves. For a dead name that is one defect printed twice;
|
|
// for a live one it inflates the very count the OK line offers as evidence.
|
|
const line = '[git.fromaitochitta.com/open/nonesuch](https://git.fromaitochitta.com/open/nonesuch)';
|
|
const f = checkLinks({ files: { 'README.md': line } }, REGISTER);
|
|
assert.equal(f.length, 1);
|
|
assert.equal(f[0].code, 'LINK-DEAD');
|
|
});
|
|
|
|
test('the OK count does not double-count a self-linking URL', () => {
|
|
const line = '[git.fromaitochitta.com/open/llm-security](https://git.fromaitochitta.com/open/llm-security)';
|
|
const f = checkLinks({ files: { 'README.md': line } }, REGISTER);
|
|
assert.equal(f[0].level, 'OK');
|
|
assert.match(f[0].msg, /^1 /);
|
|
});
|
|
|
|
test('two DIFFERENT names on one line stay two references', () => {
|
|
// The dedup key is name-and-line, not line — collapsing by line alone would
|
|
// hide the second dead name behind the first.
|
|
const line = 'https://git.fromaitochitta.com/open/nonesuch vs https://git.fromaitochitta.com/open/alsonone';
|
|
const f = checkLinks({ files: { 'README.md': line } }, REGISTER);
|
|
assert.equal(f.length, 2);
|
|
});
|
|
|
|
test('the same name on two DIFFERENT lines stays two references', () => {
|
|
const text = 'https://git.fromaitochitta.com/open/nonesuch\nand again https://git.fromaitochitta.com/open/nonesuch';
|
|
const f = checkLinks({ files: { 'README.md': text } }, REGISTER);
|
|
assert.equal(f.length, 2);
|
|
});
|
|
|
|
test('checkLinks prints no OK beside a finding, not even a WARN-only one', () => {
|
|
const dead = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/nonesuch' } }, REGISTER);
|
|
const warn = checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/coord' } }, REGISTER);
|
|
assert.equal(dead.filter((x) => x.level === 'OK').length, 0);
|
|
assert.equal(warn.filter((x) => x.level === 'OK').length, 0);
|
|
});
|
|
|
|
// ------------------------------------------------------------- description
|
|
|
|
test('description length is bounded, measured in codepoints', () => {
|
|
assert.equal(checkDescription('a fine description', REGISTER)[0].level, 'OK');
|
|
assert.equal(checkDescription('', REGISTER)[0].level, 'ERROR');
|
|
assert.equal(checkDescription('x'.repeat(181), REGISTER)[0].level, 'ERROR');
|
|
assert.equal(checkDescription('x'.repeat(180), REGISTER)[0].level, 'OK');
|
|
});
|
|
|
|
test('an unavailable description is SKIP, never a pass', () => {
|
|
// Offline is not compliance. A check that could not run says so.
|
|
const f = checkDescription(null, REGISTER);
|
|
assert.equal(f[0].level, 'SKIP');
|
|
});
|
|
|
|
// ------------------------------------------------------------- first screen
|
|
|
|
test('README line 1 must be the H1, and the description line must match the forge', () => {
|
|
const readme = '# repo-mailbox\nA local mailbox for coordination.\n';
|
|
assert.equal(
|
|
checkFirstScreen({ readme, name: 'repo-mailbox', description: 'A local mailbox for coordination.' }, REGISTER)
|
|
.every((f) => f.level === 'OK'),
|
|
true,
|
|
);
|
|
});
|
|
|
|
test('a README whose opening line diverges from the description is an ERROR', () => {
|
|
const readme = '# repo-mailbox\nSomething else entirely.\n';
|
|
const f = checkFirstScreen({ readme, name: 'repo-mailbox', description: 'A local mailbox for coordination.' }, REGISTER);
|
|
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'README-DESC'), true);
|
|
});
|
|
|
|
test('first-screen description match is SKIP when the forge text is unavailable', () => {
|
|
const f = checkFirstScreen({ readme: '# x\nbody\n', name: 'x', description: null }, REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'README-DESC' && x.level === 'SKIP'), true);
|
|
});
|
|
|
|
test('a missing H1 is an ERROR', () => {
|
|
const f = checkFirstScreen({ readme: 'no heading here\n', name: 'x', description: null }, REGISTER);
|
|
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'README-H1'), true);
|
|
});
|
|
|
|
test('an H1 that differs from the repo name is a WARN, not a failure', () => {
|
|
// `# OKR for Public Sector` is a naming choice, not a defect: the thread that
|
|
// must hold is description == catalog == opening line, and the H1 is none of
|
|
// those three. Surface it; let the operator decide.
|
|
const f = checkFirstScreen({ readme: '# OKR for Public Sector\nbody\n', name: 'okr', description: null }, REGISTER);
|
|
assert.equal(f.some((x) => x.level === 'WARN' && x.code === 'README-H1'), true);
|
|
assert.equal(f.some((x) => x.level === 'ERROR'), false);
|
|
});
|
|
|
|
test('a differing H1 does not stop the description check from running', () => {
|
|
const f = checkFirstScreen({ readme: '# Nice Title\nthe description\n', name: 'x', description: 'the description' }, REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'README-DESC' && x.level === 'OK'), true);
|
|
});
|
|
|
|
// ------------------------------------------------- registered README titles
|
|
// Census 05 measured the same 6 README-H1 WARNs three rounds running, because
|
|
// a decided YES had nowhere to live. `titles` is that place: with an entry the
|
|
// H1 is OK, without one it WARNs exactly as before — so "we decided this" and
|
|
// "nobody has looked at it" stop sharing one outcome. The wanted side effect is
|
|
// that a repo nobody has ruled on stands alone once the others are registered.
|
|
|
|
test('an H1 matching the registered title is OK, not a WARN', () => {
|
|
const readme = '# LLM Security Plugin for Claude Code\nbody\n';
|
|
const f = checkFirstScreen({ readme, name: 'llm-security', description: null }, REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'README-H1' && x.level === 'OK'), true);
|
|
assert.equal(f.some((x) => x.code === 'README-H1' && x.level === 'WARN'), false);
|
|
});
|
|
|
|
test('a repo with no registered title still WARNs — deciding and not looking stay different outcomes', () => {
|
|
const f = checkFirstScreen({ readme: '# Interaction Awareness\nbody\n', name: 'repo-mailbox', description: null }, REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'README-H1' && x.level === 'WARN'), true);
|
|
});
|
|
|
|
test('an H1 matching neither the name nor the registered title WARNs, and the message names both', () => {
|
|
const f = checkFirstScreen({ readme: '# Something Drifted\nbody\n', name: 'llm-security', description: null }, REGISTER);
|
|
const warn = f.find((x) => x.code === 'README-H1' && x.level === 'WARN');
|
|
assert.ok(warn, 'expected a README-H1 WARN');
|
|
assert.match(warn.msg, /Something Drifted/);
|
|
assert.match(warn.msg, /LLM Security Plugin for Claude Code/);
|
|
});
|
|
|
|
test('the repo name still passes when a title is registered — both spellings are accepted', () => {
|
|
const f = checkFirstScreen({ readme: '# llm-security\nbody\n', name: 'llm-security', description: null }, REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'README-H1' && x.level === 'OK'), true);
|
|
});
|
|
|
|
// --------------------------------------------- org-profile README-DESC exemption
|
|
// For an ordinary repo the README opening and the forge description describe the
|
|
// SAME subject, and equality is the right demand. For `org-profile` they do not:
|
|
// the README is the organisation's landing page, the forge description describes
|
|
// the repo. Both texts are correct about their own subject. The description field
|
|
// also renders as PLAIN TEXT, so a landing page's opening link could only ever
|
|
// match by putting raw markdown on a real surface.
|
|
const PROFILE_README = '# .profile\nCompanion repos, plugins, and tools for [From AI to Chitta](https://fromaitochitta.com).\n';
|
|
|
|
test('org-profile is exempt from the README-DESC equality check', () => {
|
|
const f = checkFirstScreen(
|
|
{ readme: PROFILE_README, name: '.profile', klass: 'org-profile', description: 'Organization profile and navigation for the open org.' },
|
|
REGISTER,
|
|
);
|
|
const desc = f.find((x) => x.code === 'README-DESC');
|
|
assert.ok(desc, 'expected a README-DESC finding — an exemption must still be recorded');
|
|
assert.equal(desc.level, 'OK');
|
|
assert.equal(f.some((x) => x.level === 'ERROR'), false);
|
|
});
|
|
|
|
test('the exemption is per class, not global — the same README under a plugin class is still an ERROR', () => {
|
|
const f = checkFirstScreen(
|
|
{ readme: PROFILE_README, name: 'llm-security', klass: 'plugin', description: 'Organization profile and navigation for the open org.' },
|
|
REGISTER,
|
|
);
|
|
assert.equal(f.some((x) => x.code === 'README-DESC' && x.level === 'ERROR'), true);
|
|
});
|
|
|
|
test('the exemption does not silence the H1 check for org-profile', () => {
|
|
const f = checkFirstScreen(
|
|
{ readme: 'no heading here\n', name: '.profile', klass: 'org-profile', description: 'anything' },
|
|
REGISTER,
|
|
);
|
|
assert.equal(f.some((x) => x.code === 'README-H1' && x.level === 'ERROR'), true);
|
|
});
|
|
|
|
// ------------------------------------------------------------ install block
|
|
|
|
const MKT = REGISTER.marketplace;
|
|
|
|
test('plugin install needs BOTH lines: marketplace add and a CLI install command', () => {
|
|
const readme = [
|
|
'## Install',
|
|
'```',
|
|
`claude plugin marketplace add ${MKT.url}`,
|
|
`claude plugin install repo-mailbox@${MKT.name}`,
|
|
'```',
|
|
].join('\n');
|
|
const f = checkInstallBlock({ readme, name: 'repo-mailbox', klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('the slash form counts as the CLI command', () => {
|
|
const readme = [
|
|
'## Install',
|
|
`claude plugin marketplace add ${MKT.url}`,
|
|
`/plugin install claude-design@${MKT.name}`,
|
|
].join('\n');
|
|
const f = checkInstallBlock({ readme, name: 'claude-design', klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('enabledPlugins JSON is an ALLOWED ADDITION, never a replacement for the CLI command', () => {
|
|
// This is the corrected defect A: 7 of 11 plugin READMEs stop after
|
|
// `marketplace add`. The JSON form works and stands in 10 of 11 — what is
|
|
// missing is a CLI command, so the contract requires the command and permits
|
|
// the JSON alongside it.
|
|
const jsonOnly = [
|
|
'## Install',
|
|
`claude plugin marketplace add ${MKT.url}`,
|
|
'Or enable directly in `~/.claude/settings.json`:',
|
|
`"enabledPlugins": { "llm-security@${MKT.name}": true }`,
|
|
].join('\n');
|
|
const f = checkInstallBlock({ readme: jsonOnly, name: 'llm-security', klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'INSTALL-NO-CLI'), true);
|
|
|
|
const both = jsonOnly + `\nclaude plugin install llm-security@${MKT.name}\n`;
|
|
const g = checkInstallBlock({ readme: both, name: 'llm-security', klass: 'plugin' }, REGISTER);
|
|
assert.equal(g.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('a missing marketplace add is its own finding — okr and claude-design are opposite halves', () => {
|
|
const noAdd = ['## Install', `/plugin install claude-design@${MKT.name}`].join('\n');
|
|
const f = checkInstallBlock({ readme: noAdd, name: 'claude-design', klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'INSTALL-NO-MARKETPLACE'), true);
|
|
|
|
// okr: neither line. Both findings fire — a rule saying "both lines" must not
|
|
// hit this repo blind.
|
|
const neither = ['## Install', `"enabledPlugins": { "okr@${MKT.name}": true }`].join('\n');
|
|
const g = checkInstallBlock({ readme: neither, name: 'okr', klass: 'plugin' }, REGISTER);
|
|
assert.equal(g.some((x) => x.code === 'INSTALL-NO-MARKETPLACE'), true);
|
|
assert.equal(g.some((x) => x.code === 'INSTALL-NO-CLI'), true);
|
|
});
|
|
|
|
test('the install target must name THIS repo, not another plugin', () => {
|
|
const readme = [
|
|
'## Install',
|
|
`claude plugin marketplace add ${MKT.url}`,
|
|
`claude plugin install some-other-plugin@${MKT.name}`,
|
|
].join('\n');
|
|
const f = checkInstallBlock({ readme, name: 'repo-standard', klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'INSTALL-NO-CLI'), true);
|
|
});
|
|
|
|
test('ssh in the marketplace add line is an ERROR — marketplace add rejects it', () => {
|
|
// Measured end-to-end: `marketplace add ssh://...` → "Invalid git URL".
|
|
// The forge UI's clone button hands you exactly that URL.
|
|
const readme = [
|
|
'## Install',
|
|
'claude plugin marketplace add ssh://git@git.fromaitochitta.com/open/ktg-plugin-marketplace.git',
|
|
`claude plugin install repo-standard@${MKT.name}`,
|
|
].join('\n');
|
|
const f = checkInstallBlock({ readme, name: 'repo-standard', klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'INSTALL-SSH'), true);
|
|
});
|
|
|
|
test('the install block is parametric — a different marketplace passes on its own values', () => {
|
|
// wiki-advise lives on the private ktg/ namespace and is distributed via
|
|
// `ktg-privat`. A skill that hardcodes the public marketplace produces an
|
|
// install line that does not work there — and being public itself, this skill
|
|
// cannot carry private marketplace names.
|
|
const priv = {
|
|
...REGISTER,
|
|
marketplace: { name: 'ktg-privat', url: 'https://git.fromaitochitta.com/ktg/ktg-privat.git' },
|
|
};
|
|
const readme = [
|
|
'## Install',
|
|
'claude plugin marketplace add https://git.fromaitochitta.com/ktg/ktg-privat.git',
|
|
'claude plugin install wiki-advise@ktg-privat',
|
|
].join('\n');
|
|
const f = checkInstallBlock({ readme, name: 'wiki-advise', klass: 'plugin' }, priv);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('the catalog needs only marketplace add — it IS the marketplace', () => {
|
|
const readme = `## Install\nclaude plugin marketplace add ${MKT.url}`;
|
|
const f = checkInstallBlock({ readme, name: 'ktg-plugin-marketplace', klass: 'catalog' }, REGISTER);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('a shared asset is vendored, not installed — a plugin install line is wrong there', () => {
|
|
const asPlugin = `## Install\nclaude plugin install playground-design-system@${MKT.name}`;
|
|
const f = checkInstallBlock(
|
|
{ readme: asPlugin, name: 'playground-design-system', klass: 'shared-asset' },
|
|
REGISTER,
|
|
);
|
|
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'INSTALL-WRONG-FORM'), true);
|
|
});
|
|
|
|
test('.profile needs no install section at all', () => {
|
|
const f = checkInstallBlock({ readme: '# .profile\nOrg profile.\n', name: '.profile', klass: 'org-profile' }, REGISTER);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
// ----------------------------------------------------------- required files
|
|
|
|
test('required files are per class, not flat across the org', () => {
|
|
const ok = checkRequiredFiles({ present: ['README.md'], klass: 'org-profile' }, REGISTER);
|
|
assert.equal(ok.filter((f) => f.level === 'ERROR').length, 0);
|
|
|
|
const missing = checkRequiredFiles({ present: ['README.md'], klass: 'plugin' }, REGISTER);
|
|
assert.equal(missing.some((f) => f.code === 'FILE-MISSING' && f.msg.includes('LICENSE')), true);
|
|
});
|
|
|
|
// Was tautological: it read `required_files` off the test's OWN local
|
|
// REGISTER fixture, so it could only ever check that this file agreed with
|
|
// itself — a typo in the real register/repos.json would drift silently past
|
|
// it. Reading the live register makes it an actual regression guard.
|
|
test('no class requires a ROADMAP — it is 0/18 and belongs to a later step', () => {
|
|
const live = loadRegister();
|
|
for (const klass of Object.keys(live.classes)) {
|
|
assert.equal(live.classes[klass].required_files.includes('ROADMAP.md'), false);
|
|
}
|
|
});
|
|
|
|
// A register row whose class does not exist reads as REGISTERED but runs the
|
|
// same zero checks as an unregistered one — the silent-loss shape this gate is
|
|
// built to catch, hiding inside a file that looks maintained. Reads the live
|
|
// register so every future row is guarded, not just today's.
|
|
test('every registered repo names a class that actually exists', () => {
|
|
const live = loadRegister();
|
|
const classes = Object.keys(live.classes);
|
|
for (const [repo, klass] of Object.entries(live.repos)) {
|
|
assert.equal(classes.includes(klass), true, `${repo} is registered as unknown class \`${klass}\``);
|
|
}
|
|
});
|
|
|
|
// -------------------------------------------------------------- aggregation
|
|
|
|
test('levelOf ranks ERROR above WARN above SKIP above OK', () => {
|
|
assert.equal(levelOf([{ level: 'OK' }, { level: 'WARN' }, { level: 'ERROR' }]), 'ERROR');
|
|
assert.equal(levelOf([{ level: 'OK' }, { level: 'WARN' }]), 'WARN');
|
|
assert.equal(levelOf([{ level: 'OK' }, { level: 'SKIP' }]), 'SKIP');
|
|
assert.equal(levelOf([{ level: 'OK' }]), 'OK');
|
|
assert.equal(levelOf([]), 'OK');
|
|
});
|
|
|
|
test('an unregistered repo is SKIP, not a pass — the gate refuses to guess a class', () => {
|
|
const r = classifyRepo({ name: 'stranger', files: {}, present: [], description: null }, REGISTER);
|
|
assert.equal(r.status, 'SKIP');
|
|
});
|
|
|
|
test('a fully compliant plugin repo classifies OK', () => {
|
|
const readme = [
|
|
'# repo-mailbox',
|
|
'A local mailbox for coordination.',
|
|
'',
|
|
'Body text.',
|
|
'',
|
|
'',
|
|
'',
|
|
'## Install',
|
|
`claude plugin marketplace add ${MKT.url}`,
|
|
`claude plugin install repo-mailbox@${MKT.name}`,
|
|
'',
|
|
'## Non-goals',
|
|
'It is not a state store.',
|
|
'',
|
|
'## Changelog',
|
|
'See [CHANGELOG.md](CHANGELOG.md).',
|
|
].join('\n');
|
|
const r = classifyRepo(
|
|
{
|
|
name: 'repo-mailbox',
|
|
files: { 'README.md': readme },
|
|
present: ['README.md', 'LICENSE', 'CHANGELOG.md', '.claude-plugin/plugin.json'],
|
|
description: 'A local mailbox for coordination.',
|
|
pluginVersion: '0.7.0',
|
|
readmeBadge: '0.7.0',
|
|
changelogTop: '0.7.0',
|
|
tags: ['v0.7.0'],
|
|
catalogNames: ['repo-mailbox'],
|
|
},
|
|
REGISTER,
|
|
);
|
|
assert.equal(r.status, 'OK');
|
|
assert.deepEqual(r.buckets, { broken: 0, missing: 0, weakening: 0 });
|
|
});
|
|
|
|
test('the security trait travels from the register into the verdict', () => {
|
|
const r = classifyRepo(
|
|
{
|
|
name: 'llm-security',
|
|
files: { 'README.md': '# llm-security\ndesc\n## Install\n## Non-goals\n## Changelog\n' },
|
|
present: ['README.md', 'LICENSE', 'CHANGELOG.md', '.claude-plugin/plugin.json'],
|
|
description: 'desc',
|
|
},
|
|
REGISTER,
|
|
);
|
|
assert.deepEqual(r.traits, ['security']);
|
|
assert.equal(r.findings.some((f) => f.code === 'FILE-MISSING' && f.msg.includes('SECURITY.md')), true);
|
|
assert.equal(r.findings.some((f) => f.code === 'HEADING-MISSING' && f.msg.includes('Known limitations')), true);
|
|
});
|
|
|
|
// =========================================================================
|
|
// The brief's reporting contract: findings carry a BUCKET alongside a level.
|
|
// "broken now" (strangers are blocked or misled), "missing" (an expected
|
|
// artefact is absent), "weakening" (present, but reads as amateur). The two
|
|
// axes are independent: a weakening finding can still be an ERROR.
|
|
// =========================================================================
|
|
|
|
test('every ERROR and WARN finding carries a bucket', () => {
|
|
const readme = ['## Install', `"enabledPlugins": { "okr@${MKT.name}": true }`].join('\n');
|
|
const all = [
|
|
...checkInstallBlock({ readme, name: 'okr', klass: 'plugin' }, REGISTER),
|
|
...checkLinks({ files: { 'README.md': 'https://git.fromaitochitta.com/open/nonesuch' } }, REGISTER),
|
|
...checkRequiredFiles({ present: ['README.md'], klass: 'plugin' }, REGISTER),
|
|
...checkDescription('', REGISTER),
|
|
];
|
|
const graded = all.filter((f) => f.level === 'ERROR' || f.level === 'WARN');
|
|
assert.ok(graded.length > 0);
|
|
for (const f of graded) {
|
|
assert.ok(['broken', 'missing', 'weakening'].includes(f.bucket), `${f.code} has bucket ${f.bucket}`);
|
|
}
|
|
});
|
|
|
|
test('an unusable install path is `broken`, an absent file is `missing`', () => {
|
|
const readme = ['## Install', 'nothing useful here'].join('\n');
|
|
const inst = checkInstallBlock({ readme, name: 'okr', klass: 'plugin' }, REGISTER);
|
|
assert.equal(inst.find((f) => f.code === 'INSTALL-NO-CLI').bucket, 'broken');
|
|
|
|
const files = checkRequiredFiles({ present: ['README.md'], klass: 'plugin' }, REGISTER);
|
|
assert.equal(files.find((f) => f.code === 'FILE-MISSING').bucket, 'missing');
|
|
});
|
|
|
|
// ------------------------------------------------------- version consistency
|
|
|
|
test('version consistency: manifest, README badge and CHANGELOG must agree', () => {
|
|
const ok = checkVersionConsistency({ pluginVersion: '0.1.0', readmeBadge: '0.1.0', changelogTop: '0.1.0', tags: ['v0.1.0'] });
|
|
assert.equal(ok.filter((f) => f.level === 'ERROR').length, 0);
|
|
|
|
const drift = checkVersionConsistency({ pluginVersion: '0.1.0', readmeBadge: '0.0.9', changelogTop: '0.1.0', tags: ['v0.1.0'] });
|
|
assert.equal(drift.some((f) => f.level === 'ERROR' && f.code === 'VERSION-BADGE'), true);
|
|
|
|
const stale = checkVersionConsistency({ pluginVersion: '0.2.0', readmeBadge: '0.2.0', changelogTop: '0.1.0', tags: ['v0.1.0', 'v0.2.0'] });
|
|
assert.equal(stale.some((f) => f.level === 'ERROR' && f.code === 'VERSION-CHANGELOG'), true);
|
|
});
|
|
|
|
test('an untagged repo SKIPs the tag comparison rather than failing it', () => {
|
|
// Nothing has been released yet. That is a state, not a defect — and "SKIP is
|
|
// never a pass" means it must say so rather than quietly succeed.
|
|
const f = checkVersionConsistency({ pluginVersion: '0.1.0', readmeBadge: '0.1.0', changelogTop: '0.1.0', tags: [] });
|
|
assert.equal(f.some((x) => x.code === 'VERSION-TAG' && x.level === 'SKIP'), true);
|
|
assert.equal(f.some((x) => x.level === 'ERROR'), false);
|
|
});
|
|
|
|
test('a released version with no matching tag is an ERROR', () => {
|
|
const f = checkVersionConsistency({ pluginVersion: '0.3.1', readmeBadge: '0.3.1', changelogTop: '0.3.1', tags: ['v0.1.0'] });
|
|
assert.equal(f.some((x) => x.level === 'ERROR' && x.code === 'VERSION-TAG'), true);
|
|
});
|
|
|
|
// Reported by llm-ingestion-okf (coord, 2026-08-03): a PEP 440 pre-release
|
|
// (`0.5.0a2`) matches the manifest and the tag exactly, but the CHANGELOG
|
|
// extractor truncated it to `0.5.0` — the only way to reach 0 ERROR would have
|
|
// been to announce a release that never happened.
|
|
test('extractChangelogTop keeps a PEP 440 pre-release suffix, not just X.Y.Z', () => {
|
|
assert.equal(extractChangelogTop('## [0.5.0a2] - 2026-07-31'), '0.5.0a2');
|
|
assert.equal(extractChangelogTop('## [0.1.1] — 2026-08-03'), '0.1.1');
|
|
});
|
|
|
|
test('a PEP 440 pre-release version agrees with its own CHANGELOG heading', () => {
|
|
const f = checkVersionConsistency({
|
|
pluginVersion: '0.5.0a2', readmeBadge: '0.5.0a2', changelogTop: extractChangelogTop('## [0.5.0a2] - 2026-07-31'), tags: ['v0.5.0a2'],
|
|
});
|
|
assert.equal(f.some((x) => x.level === 'ERROR'), false);
|
|
});
|
|
|
|
// -------------------------------------------------------- required headings
|
|
|
|
test('required headings are per class — Non-goals is required, not optional', () => {
|
|
const full = '# x\n## Install\n## Non-goals\n## Changelog\n';
|
|
assert.equal(checkHeadings({ readme: full, klass: 'plugin' }, REGISTER).filter((f) => f.level === 'ERROR').length, 0);
|
|
|
|
const noNonGoals = '# x\n## Install\n## Changelog\n';
|
|
const f = checkHeadings({ readme: noNonGoals, klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'HEADING-MISSING' && x.msg.includes('Non-goals')), true);
|
|
assert.equal(f.find((x) => x.code === 'HEADING-MISSING').bucket, 'missing');
|
|
});
|
|
|
|
test('org-profile requires no headings at all', () => {
|
|
const f = checkHeadings({ readme: '# .profile\nprofile\n', klass: 'org-profile' }, REGISTER);
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
// ------------------------------------------------------------ badge honesty
|
|
|
|
test('a static badge asserting test or build status is a finding', () => {
|
|
// "tests: 642 passing" as a static image is a claim dressed as evidence.
|
|
const f = checkBadges({ readme: '' });
|
|
assert.equal(f.some((x) => x.code === 'BADGE-STATIC-CLAIM'), true);
|
|
assert.equal(f.find((x) => x.code === 'BADGE-STATIC-CLAIM').bucket, 'weakening');
|
|
});
|
|
|
|
test('static version, licence and platform badges are fine — they assert no run', () => {
|
|
const readme = [
|
|
'',
|
|
'',
|
|
'',
|
|
].join('\n');
|
|
assert.equal(checkBadges({ readme }).filter((f) => f.level !== 'OK').length, 0);
|
|
});
|
|
|
|
test('a build badge that links to a real run is fine', () => {
|
|
const readme = '[](https://forge.example/repo/actions)';
|
|
assert.equal(checkBadges({ readme }).filter((f) => f.level !== 'OK').length, 0);
|
|
});
|
|
|
|
// The gap: being LINKED was treated as proof of a real run, but nothing ever
|
|
// checked that the link actually went anywhere. A badge linked to a dead
|
|
// relative path is a claim dressed as evidence that LOOKS more credible than
|
|
// a static one, not less. External targets (the ordinary case — a CI
|
|
// provider) still need the network and stay out of scope, same as
|
|
// checkInternalLinks.
|
|
test('a linked badge whose relative target does not exist is a finding, not silently fine', () => {
|
|
const readme = '[](docs/ci-results.md)';
|
|
const f = checkBadges({ readme, present: ['README.md'] });
|
|
assert.equal(f.some((x) => x.code === 'BADGE-DEAD-LINK'), true);
|
|
const hit = f.find((x) => x.code === 'BADGE-DEAD-LINK');
|
|
assert.equal(hit.level, 'ERROR');
|
|
assert.equal(hit.bucket, 'broken');
|
|
});
|
|
|
|
test('a linked badge whose relative target exists is fine', () => {
|
|
const readme = '[](docs/ci-results.md)';
|
|
const f = checkBadges({ readme, present: ['README.md', 'docs/ci-results.md'] });
|
|
assert.equal(f.some((x) => x.code === 'BADGE-DEAD-LINK'), false);
|
|
});
|
|
|
|
test('a linked badge with no `present` given is not falsely flagged dead — external targets stay unverifiable', () => {
|
|
// Guards the call site: checkBadges must still tolerate being called
|
|
// without `present`, same as checkInternalLinks tolerates it.
|
|
const readme = '[](https://forge.example/repo/actions)';
|
|
assert.equal(checkBadges({ readme }).some((x) => x.code === 'BADGE-DEAD-LINK'), false);
|
|
});
|
|
|
|
// Reported by llm-ingestion-pipeline-security (coord, 2026-08-03): a bare
|
|
// `status` badge is a self-declared maturity label ("alpha", "experimental"),
|
|
// the same class as version/licence/platform, which already assert no run —
|
|
// not a run claim like "build status" or "CI status".
|
|
test('a static maturity-status badge asserts no run, unlike build/CI status', () => {
|
|
const f = checkBadges({ readme: '' });
|
|
assert.equal(f.some((x) => x.code === 'BADGE-STATIC-CLAIM'), false);
|
|
});
|
|
|
|
test('a build- or CI-status badge is still caught — only bare "status" was too wide', () => {
|
|
const build = checkBadges({ readme: '' });
|
|
assert.equal(build.some((x) => x.code === 'BADGE-STATIC-CLAIM'), true);
|
|
const ci = checkBadges({ readme: '' });
|
|
assert.equal(ci.some((x) => x.code === 'BADGE-STATIC-CLAIM'), true);
|
|
});
|
|
|
|
// ----------------------------------------------------------- badge crowding
|
|
|
|
// Trockman et al., ICSE 2018 (n=294,941 npm packages) measured a non-linear
|
|
// relationship between badge count and popularity with a predicted inflection
|
|
// at five, motivated by survey respondents calling over-badged READMEs
|
|
// cluttered and "trying too hard". The coefficient sits in an appendix with no
|
|
// CI or p-value, so this is a WARN and the threshold is the measured inflection
|
|
// — not a rounder number that would read as invented.
|
|
const badges = (n, wrap = (s) => s) =>
|
|
Array.from({ length: n }, (_, i) => wrap(``)).join('\n');
|
|
|
|
test('more than five badges is a finding — the measured inflection point', () => {
|
|
const f = checkBadges({ readme: badges(6) });
|
|
const hit = f.find((x) => x.code === 'BADGE-COUNT');
|
|
assert.ok(hit, 'six badges should produce BADGE-COUNT');
|
|
assert.equal(hit.level, 'WARN');
|
|
assert.equal(hit.bucket, 'weakening');
|
|
});
|
|
|
|
test('exactly five badges is not a finding — the source cannot carry a harder rule', () => {
|
|
assert.equal(checkBadges({ readme: badges(5) }).some((x) => x.code === 'BADGE-COUNT'), false);
|
|
});
|
|
|
|
test('a linked badge still counts toward the total — linking answers honesty, not clutter', () => {
|
|
const readme = badges(6, (s) => `[${s}](https://forge.example/run)`);
|
|
assert.equal(checkBadges({ readme }).some((x) => x.code === 'BADGE-COUNT'), true);
|
|
});
|
|
|
|
test('content images are not badges, however many there are', () => {
|
|
const readme = Array.from({ length: 9 }, (_, i) => ``).join('\n');
|
|
assert.equal(checkBadges({ readme }).some((x) => x.code === 'BADGE-COUNT'), false);
|
|
});
|
|
|
|
// ---------------------------------------------------------- README language
|
|
|
|
// The operator owns this axis, exactly as they own `traits`. English is the
|
|
// default; a repo aimed ONLY at a Norwegian readership is declared `nb` and is
|
|
// then WRONG in English, not right. Measured 2026-08-03: `ms-ai-architect` and
|
|
// `okr` are the two such repos, and both currently carry English prose.
|
|
const LOCALE_REGISTER = { ...REGISTER, locales: { okr: 'nb' } };
|
|
|
|
const NB_PROSE = [
|
|
'# okr',
|
|
'',
|
|
'Dette er en plugin som ikke gjør noe annet enn å måle mål og resultater.',
|
|
'Den kan kjøres fra Claude Code, og den skal være tilgjengelig når du',
|
|
'trenger den. Hvis du vil ha mer, se dokumentasjonen. Alle kommandoer',
|
|
'blir kjørt fra en aktiv sesjon, og ingenting av dette krever en server.',
|
|
].join('\n');
|
|
|
|
const EN_PROSE = [
|
|
'# okr',
|
|
'',
|
|
'This is a plugin that does nothing other than measure objectives and',
|
|
'results. It can be run from Claude Code, and it is always available when',
|
|
'you need it. If you want more, see the documentation. All of the commands',
|
|
'are run from an active session, and none of this requires a server.',
|
|
].join('\n');
|
|
|
|
test('a repo declared Norwegian that ships English prose is a finding', () => {
|
|
const f = checkReadmeLanguage({ readme: EN_PROSE, name: 'okr' }, LOCALE_REGISTER);
|
|
const hit = f.find((x) => x.code === 'README-LANGUAGE');
|
|
assert.ok(hit, 'declared nb + English prose should produce README-LANGUAGE');
|
|
assert.equal(hit.level, 'WARN');
|
|
assert.equal(hit.bucket, 'weakening');
|
|
});
|
|
|
|
test('a repo declared Norwegian that ships Norwegian prose passes', () => {
|
|
const f = checkReadmeLanguage({ readme: NB_PROSE, name: 'okr' }, LOCALE_REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'README-LANGUAGE'), false);
|
|
});
|
|
|
|
test('an undeclared repo defaults to English, so Norwegian prose is the finding', () => {
|
|
const f = checkReadmeLanguage({ readme: NB_PROSE, name: 'repo-standard' }, LOCALE_REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'README-LANGUAGE'), true);
|
|
});
|
|
|
|
test('an undeclared repo shipping English prose passes', () => {
|
|
const f = checkReadmeLanguage({ readme: EN_PROSE, name: 'repo-standard' }, LOCALE_REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'README-LANGUAGE'), false);
|
|
});
|
|
|
|
// Same discipline as the link and boilerplate checks: a Norwegian identifier in
|
|
// a shell example must not decide what language the DOCUMENT is written in.
|
|
test('code blocks do not decide the language', () => {
|
|
const readme = [
|
|
EN_PROSE,
|
|
'',
|
|
'```bash',
|
|
'kjør --og --ikke --som --det --den --er --på --til --av --med --om',
|
|
'kjør --har --kan --skal --blir --etter --når --også --hvis --eller',
|
|
'```',
|
|
].join('\n');
|
|
const f = checkReadmeLanguage({ readme, name: 'repo-standard' }, LOCALE_REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'README-LANGUAGE'), false);
|
|
});
|
|
|
|
// A README with no running prose has no language to be wrong about — the check
|
|
// ran and found nothing, which is not the same as a check that could not run.
|
|
// Routing it to SKIP would mean no terse repo could ever classify OK.
|
|
test('no running prose is an OK, not a SKIP — nothing claims a language', () => {
|
|
const f = checkReadmeLanguage({ readme: '# thing\n\nA tool.\n', name: 'repo-standard' }, LOCALE_REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'README-LANGUAGE'), false);
|
|
assert.equal(f[0].level, 'OK');
|
|
});
|
|
|
|
// A document with real prose that the gate genuinely cannot call IS a SKIP —
|
|
// here the question is live and unanswered, unlike the empty case above.
|
|
test('prose that mixes languages too evenly is a SKIP, never a pass', () => {
|
|
const readme = [NB_PROSE, EN_PROSE].join('\n\n');
|
|
const f = checkReadmeLanguage({ readme, name: 'repo-standard' }, LOCALE_REGISTER);
|
|
const hit = f.find((x) => x.code === 'README-LANGUAGE-UNDECIDABLE');
|
|
assert.ok(hit, 'an evenly bilingual README cannot be called');
|
|
assert.equal(hit.level, 'SKIP');
|
|
});
|
|
|
|
// -------------------------------------------------------------- boilerplate
|
|
|
|
test('unfinished template text is a finding', () => {
|
|
const f = checkBoilerplate({ files: { 'README.md': 'Install your-project-name today' } });
|
|
assert.equal(f.some((x) => x.code === 'BOILERPLATE'), true);
|
|
|
|
const g = checkBoilerplate({ files: { 'CODE_OF_CONDUCT.md': 'Report to [INSERT EMAIL ADDRESS]' } });
|
|
assert.equal(g.some((x) => x.code === 'BOILERPLATE'), true);
|
|
});
|
|
|
|
test('a lone FIXME with no TODO alongside is still caught', () => {
|
|
const f = checkBoilerplate({ files: { 'NOTES.md': 'FIXME: handle the null case here.' } });
|
|
assert.equal(f.some((x) => x.level === 'WARN'), true);
|
|
});
|
|
|
|
// Reported by config-audit (coord, 2026-08-03): a scanner whose JOB is to find
|
|
// TODO/FIXME markers in OTHER repos names its own detection target in its own
|
|
// docs — prose and a table row, neither wrapped in backticks. "TODO/FIXME"
|
|
// named together is the convention itself, not a forgotten instance of one.
|
|
test('"TODO/FIXME" named together as the convention is not a live marker', () => {
|
|
const listItem = checkBoilerplate({
|
|
files: { 'agents/scanner-agent.md': "- Flag TODO/FIXME markers that haven't been addressed" },
|
|
});
|
|
assert.equal(listItem.some((x) => x.level === 'WARN'), false);
|
|
|
|
const tableRow = checkBoilerplate({
|
|
files: { 'knowledge/anti-patterns.md': '| 5 | TODO/FIXME comments in CLAUDE.md | CA-CML-005 | low |' },
|
|
});
|
|
assert.equal(tableRow.some((x) => x.level === 'WARN'), false);
|
|
});
|
|
|
|
test('ordinary prose is not boilerplate', () => {
|
|
const f = checkBoilerplate({ files: { 'README.md': 'This project solves a real problem.' } });
|
|
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
|
|
});
|
|
|
|
// --------------------------------------------------- licence claim vs. file
|
|
|
|
test('a README that cites a LICENSE the repo does not have is broken', () => {
|
|
const f = checkLicenseClaim({ readme: 'Released under the [MIT licence](LICENSE).', present: ['README.md'] });
|
|
assert.equal(f.some((x) => x.code === 'LICENSE-CLAIMED-ABSENT' && x.bucket === 'broken'), true);
|
|
});
|
|
|
|
test('a cited LICENSE that exists passes', () => {
|
|
const f = checkLicenseClaim({ readme: 'See [LICENSE](LICENSE).', present: ['README.md', 'LICENSE'] });
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
// ------------------------------------------------------------ internal links
|
|
|
|
test('a relative link to a file that does not exist is a finding', () => {
|
|
const f = checkInternalLinks(
|
|
{ files: { 'README.md': 'See [the design](docs/design.md).' }, present: ['README.md'] },
|
|
);
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), true);
|
|
});
|
|
|
|
test('relative links that resolve, anchors and external URLs are left alone', () => {
|
|
const f = checkInternalLinks({
|
|
files: { 'README.md': '[a](docs/design.md) [b](#section) [c](https://example.com) [d](mailto:x@y.z)' },
|
|
present: ['README.md', 'docs/design.md'],
|
|
});
|
|
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
|
|
});
|
|
|
|
// ----------------------------------------------------------------- traits
|
|
|
|
test('the security trait adds a SECURITY.md requirement on top of the class', () => {
|
|
const f = checkRequiredFiles({ present: ['README.md', 'LICENSE'], klass: 'standalone', traits: ['security'] }, REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'FILE-MISSING' && x.msg.includes('SECURITY.md')), true);
|
|
});
|
|
|
|
test('a repo without the security trait owes no SECURITY.md', () => {
|
|
const f = checkRequiredFiles({ present: ['README.md', 'LICENSE'], klass: 'standalone' }, REGISTER);
|
|
assert.equal(f.some((x) => x.msg.includes('SECURITY.md')), false);
|
|
});
|
|
|
|
test('the security trait requires limitations to be stated', () => {
|
|
const f = checkHeadings({ readme: '# x\n## Install\n## Non-goals\n', klass: 'standalone', traits: ['security'] }, REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'HEADING-MISSING' && x.msg.includes('Known limitations')), true);
|
|
});
|
|
|
|
// Same fix as the ROADMAP test above: was checking the test's own fixture
|
|
// against itself. Reading the live register makes it a real guard again.
|
|
test('CONTRIBUTING and CODE_OF_CONDUCT are required by no class — the maintainer works alone', () => {
|
|
const live = loadRegister();
|
|
for (const klass of Object.keys(live.classes)) {
|
|
const req = live.classes[klass].required_files;
|
|
assert.equal(req.includes('CONTRIBUTING.md'), false);
|
|
assert.equal(req.includes('CODE_OF_CONDUCT.md'), false);
|
|
assert.equal(req.includes('MAINTAINERS.md'), false);
|
|
}
|
|
});
|
|
|
|
// ---------------------------------------------- link check: the noise sources
|
|
// All three found by running against llm-security, which produced ~30 false
|
|
// positives on the first pass. A check that is wrong this often teaches people
|
|
// to ignore it, which is worse than not having it.
|
|
|
|
test('a regex inside an inline code span is not a markdown link', () => {
|
|
// `["']([A-Za-z0-9\-._]{16,64})["']` is `[...](...)` to a naive scanner.
|
|
const line = '- **Regex:** `(?i)\\bapi[_\\-]?key\\s*[:=]\\s*["\']([A-Za-z0-9\\-._]{16,64})["\']`';
|
|
const f = checkInternalLinks({ files: { 'k.md': line }, present: ['k.md'] });
|
|
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
|
|
});
|
|
|
|
test('fenced code blocks are not scanned for links', () => {
|
|
const text = ['```bash', 'curl [x](not-a-real-file.md)', '```'].join('\n');
|
|
const f = checkInternalLinks({ files: { 'r.md': text }, present: ['r.md'] });
|
|
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
|
|
});
|
|
|
|
test('any URI scheme is left alone, not just http', () => {
|
|
const text = '[a](file:///abs/path.html) [b](ftp://x/y) [c](vscode://z)';
|
|
const f = checkInternalLinks({ files: { 'r.md': text }, present: ['r.md'] });
|
|
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
|
|
});
|
|
|
|
test('a path that escapes the repo is unresolvable, not broken', () => {
|
|
// Legitimately common: a plugin README pointing up at its marketplace.
|
|
// The gate sees one repo, so it cannot judge — and must not pretend to.
|
|
const f = checkInternalLinks({ files: { 'README.md': '[d](../../README.md)' }, present: ['README.md'] });
|
|
assert.equal(f.some((x) => x.code === 'LINK-OUTSIDE-REPO' && x.level === 'SKIP'), true);
|
|
assert.equal(f.some((x) => x.level === 'ERROR'), false);
|
|
});
|
|
|
|
test('a genuinely missing sibling file is still an ERROR', () => {
|
|
const f = checkInternalLinks({ files: { 'README.md': '[e](docs/gone.md)' }, present: ['README.md'] });
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING' && x.level === 'ERROR'), true);
|
|
});
|
|
|
|
// Reported by portfolio-optimiser-claude (coord 20260803T194933Z): `present` is
|
|
// the set of tracked FILES, so a directory link never has a member to match,
|
|
// even when every file under it is tracked. `have` never contained a directory
|
|
// to begin with — the fix derives one from `present`, it does not loosen it.
|
|
test('a link to a directory that is genuinely tracked resolves', () => {
|
|
const f = checkInternalLinks({
|
|
files: { 'README.md': 'See [dir](runs/s10/) and [file](runs/s10/a.json).' },
|
|
present: ['README.md', 'runs/s10/a.json'],
|
|
});
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), false);
|
|
});
|
|
|
|
test('a link to a directory with no tracked files under it is still an ERROR', () => {
|
|
const f = checkInternalLinks({
|
|
files: { 'README.md': '[ghost](docs/ghost/)' },
|
|
present: ['README.md'],
|
|
});
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING' && x.level === 'ERROR'), true);
|
|
});
|
|
|
|
test('a required heading present at the wrong level says so', () => {
|
|
// llm-security has `### Install` nested under `## Quick Start`. The contract
|
|
// wants it at level 2 — the finding should name that, not just "missing".
|
|
const f = checkHeadings({ readme: '# x\n## Quick Start\n### Install\n## Non-goals\n## Changelog\n', klass: 'plugin' }, REGISTER);
|
|
const hit = f.find((x) => x.code === 'HEADING-LEVEL');
|
|
assert.ok(hit, 'expected a HEADING-LEVEL finding');
|
|
assert.match(hit.msg, /### Install/);
|
|
});
|
|
|
|
test('a document ABOUT placeholders does not trip the placeholder detector', () => {
|
|
// llm-security documents the very strings it scans for. Inside code spans and
|
|
// fenced blocks they are subject matter, not unfinished template text.
|
|
const text = [
|
|
'Skip if the matched value contains: `your-project-name`, `FIXME`, `changeme`.',
|
|
'',
|
|
'```bash',
|
|
'cd <your-fork>',
|
|
'```',
|
|
].join('\n');
|
|
const f = checkBoilerplate({ files: { 'k.md': text } });
|
|
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
|
|
});
|
|
|
|
test('placeholder text in ordinary prose is still caught', () => {
|
|
const f = checkBoilerplate({ files: { 'README.md': 'Install your-project-name to begin.' } });
|
|
assert.equal(f.some((x) => x.code === 'BOILERPLATE'), true);
|
|
});
|
|
|
|
// --------------------------------- relative links resolve against their file
|
|
|
|
test('resolveRelative joins against the containing file, not the repo root', () => {
|
|
assert.equal(resolveRelative('examples/demo/README.md', 'expected.md'), 'examples/demo/expected.md');
|
|
assert.equal(resolveRelative('README.md', 'docs/design.md'), 'docs/design.md');
|
|
assert.equal(resolveRelative('docs/a/b.md', '../c.md'), 'docs/c.md');
|
|
assert.equal(resolveRelative('docs/a.md', './b.md'), 'docs/b.md');
|
|
});
|
|
|
|
test('resolveRelative returns null when the path escapes the repo', () => {
|
|
assert.equal(resolveRelative('README.md', '../../README.md'), null);
|
|
assert.equal(resolveRelative('docs/a.md', '../../../x.md'), null);
|
|
assert.equal(resolveRelative('README.md', '/etc/passwd'), null);
|
|
});
|
|
|
|
test('a link from a nested README to its sibling resolves', () => {
|
|
// Found against llm-security: both files existed, and the gate called them
|
|
// missing because it compared a file-relative target to repo-root paths.
|
|
const f = checkInternalLinks({
|
|
files: { 'examples/demo/README.md': 'See [findings](expected-findings.md).' },
|
|
present: ['examples/demo/README.md', 'examples/demo/expected-findings.md'],
|
|
});
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('a nested link to a genuinely absent sibling is still reported', () => {
|
|
const f = checkInternalLinks({
|
|
files: { 'examples/demo/README.md': 'See [gone](gone.md).' },
|
|
present: ['examples/demo/README.md'],
|
|
});
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), true);
|
|
});
|
|
|
|
// ------------------------------------------- link level follows the reader
|
|
// Measured: 30 of 43 LINK-INTERNAL-MISSING across the org sat in `shared/`,
|
|
// `docs/plan/` and `.claude/` — path-traversal test fixtures with deliberately
|
|
// invalid targets, internal session plans, agent working files. Only 12 were in
|
|
// a README. Every one of those 30 was an ERROR, which is how a gate gets
|
|
// switched off. Who the reader is decides what is required: the repo root is
|
|
// the shop window, everything below it is internal.
|
|
|
|
test('a dead link in a root document blocks a stranger — ERROR', () => {
|
|
const f = checkInternalLinks({ files: { 'README.md': '[e](docs/gone.md)' }, present: ['README.md'] });
|
|
const hit = f.find((x) => x.code === 'LINK-INTERNAL-MISSING');
|
|
assert.equal(hit.level, 'ERROR');
|
|
assert.equal(hit.bucket, 'broken');
|
|
});
|
|
|
|
test('a dead link below the root is a WARN, not an ERROR', () => {
|
|
// `shared/examples/nav-golden-escape/` in portfolio-optimiser links at
|
|
// `../../../../etc/passwd` ON PURPOSE — it is the fixture for a path-traversal
|
|
// test. Nineteen ERRORs against that repo were almost all this.
|
|
const f = checkInternalLinks({
|
|
files: { 'docs/plan/session.md': '[x](../gone.md)' },
|
|
present: ['docs/plan/session.md'],
|
|
});
|
|
const hit = f.find((x) => x.code === 'LINK-INTERNAL-MISSING');
|
|
assert.equal(hit.level, 'WARN');
|
|
assert.equal(hit.bucket, 'broken');
|
|
});
|
|
|
|
test('only the level moves — an internal dead link is never silently dropped', () => {
|
|
const f = checkInternalLinks({
|
|
files: { 'docs/a.md': '[x](gone.md)' },
|
|
present: ['docs/a.md'],
|
|
});
|
|
assert.equal(f.filter((x) => x.code === 'LINK-INTERNAL-MISSING').length, 1);
|
|
assert.match(f.find((x) => x.code === 'LINK-INTERNAL-MISSING').msg, /docs\/a\.md:1/);
|
|
});
|
|
|
|
test('WARN-only links must not also report that every link resolves', () => {
|
|
// The OK line said "every resolvable relative link resolves" whenever there
|
|
// was no ERROR. Degrading below-root links to WARN would have made that line
|
|
// appear beside its own counter-evidence.
|
|
const f = checkInternalLinks({
|
|
files: { 'docs/a.md': '[x](gone.md)' },
|
|
present: ['docs/a.md'],
|
|
});
|
|
assert.equal(f.some((x) => x.code === 'LINKS-INTERNAL'), false);
|
|
});
|
|
|
|
// ------------------------------------ fixture paths are presumed intentional
|
|
// `shared/examples/nav-golden-escape/bundle/index.md` in portfolio-optimiser
|
|
// deliberately escapes with `../../../../etc/passwd` — the deep `..` pops the
|
|
// whole base path and lands on `etc/passwd`, a path that is not `null` (still
|
|
// inside the repo by the resolver's arithmetic) and not tracked, so it read as
|
|
// a genuine WARN. It is the fixture doing its job, not a broken link. Third
|
|
// tool in the org to hit this same pattern — the check was the thing at fault.
|
|
|
|
test('a dead link inside a *golden* fixture path is SKIP with its own code, not WARN', () => {
|
|
const f = checkInternalLinks({
|
|
files: { 'shared/examples/nav-golden-escape/bundle/index.md': '[x](../../../../etc/passwd)' },
|
|
present: ['shared/examples/nav-golden-escape/bundle/index.md'],
|
|
});
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), false);
|
|
const hit = f.find((x) => x.code === 'LINK-INTERNAL-FIXTURE');
|
|
assert.equal(hit.level, 'SKIP');
|
|
assert.equal(hit.bucket, undefined);
|
|
});
|
|
|
|
test('a dead link inside a tests/ or fixtures/ directory is SKIP, not judged', () => {
|
|
const f = checkInternalLinks({
|
|
files: { 'tests/fixtures/plan.md': '[x](gone.md)' },
|
|
present: ['tests/fixtures/plan.md'],
|
|
});
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), false);
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-FIXTURE' && x.level === 'SKIP'), true);
|
|
});
|
|
|
|
test('a fixture-path dead link is never silently dropped — SKIP still names file and line', () => {
|
|
const f = checkInternalLinks({
|
|
files: { 'tests/plan.md': '[x](gone.md)' },
|
|
present: ['tests/plan.md'],
|
|
});
|
|
const hit = f.find((x) => x.code === 'LINK-INTERNAL-FIXTURE');
|
|
assert.match(hit.msg, /tests\/plan\.md:1/);
|
|
});
|
|
|
|
test('SKIP-classified fixture links do not suppress the every-link-resolves OK line', () => {
|
|
// LINK-OUTSIDE-REPO already sits outside the LINK-INTERNAL-MISSING check that
|
|
// gates the OK line; LINK-INTERNAL-FIXTURE follows the same precedent.
|
|
const f = checkInternalLinks({
|
|
files: { 'tests/plan.md': '[x](gone.md)' },
|
|
present: ['tests/plan.md'],
|
|
});
|
|
assert.equal(f.some((x) => x.code === 'LINKS-INTERNAL'), true);
|
|
});
|
|
|
|
test('a directory only substring-matching "test" or "fixtures" is not treated as a fixture path', () => {
|
|
// Exact segment match only for the literal names — "testing/" or
|
|
// "fixturesque/" are real directories, not the fixture convention. Only
|
|
// *golden* is a deliberate substring glob.
|
|
const f = checkInternalLinks({
|
|
files: { 'testing/plan.md': '[x](gone.md)' },
|
|
present: ['testing/plan.md'],
|
|
});
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-FIXTURE'), false);
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING' && x.level === 'WARN'), true);
|
|
});
|
|
|
|
test('a root-level dead link is unaffected by the fixture heuristic', () => {
|
|
const f = checkInternalLinks({ files: { 'README.md': '[e](docs/gone.md)' }, present: ['README.md'] });
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-FIXTURE'), false);
|
|
assert.equal(f.find((x) => x.code === 'LINK-INTERNAL-MISSING').level, 'ERROR');
|
|
});
|
|
|
|
// ------------------------------------------------ the repo name is the remote
|
|
// `catalog/` is the working directory of the repo named `ktg-plugin-marketplace`.
|
|
// Deriving the name from the directory basename left it REPO-UNREGISTERED and
|
|
// ran zero checks against the one repo the whole catalog rule depends on.
|
|
|
|
test('the repo name comes from the remote, not the directory', () => {
|
|
assert.equal(
|
|
parseRepoNameFromRemote('ssh://git@git.fromaitochitta.com/open/ktg-plugin-marketplace.git'),
|
|
'ktg-plugin-marketplace',
|
|
);
|
|
assert.equal(
|
|
parseRepoNameFromRemote('https://git.fromaitochitta.com/open/repo-standard.git'),
|
|
'repo-standard',
|
|
);
|
|
});
|
|
|
|
test('remote parsing handles the scp form and a missing .git suffix', () => {
|
|
// The forge UI hands out the scp form from its clone button.
|
|
assert.equal(parseRepoNameFromRemote('git@git.fromaitochitta.com:open/okr.git'), 'okr');
|
|
assert.equal(parseRepoNameFromRemote('https://git.fromaitochitta.com/open/okr'), 'okr');
|
|
assert.equal(parseRepoNameFromRemote('https://git.fromaitochitta.com/open/okr/'), 'okr');
|
|
});
|
|
|
|
test('an absent or unparseable remote yields null, so the caller can fall back', () => {
|
|
assert.equal(parseRepoNameFromRemote(''), null);
|
|
assert.equal(parseRepoNameFromRemote(null), null);
|
|
assert.equal(parseRepoNameFromRemote(' '), null);
|
|
assert.equal(parseRepoNameFromRemote('https://git.fromaitochitta.com/'), null);
|
|
});
|
|
|
|
// ------------------------------- stripCode must not swallow nested list items
|
|
|
|
test('a link inside a nested list item is still scanned', () => {
|
|
// An indented line is only a code block when a blank line precedes it.
|
|
// Treating every 4-space indent as code made nested bullets invisible to both
|
|
// the link and boilerplate checks — a silent false pass, which is worse than
|
|
// the noise it was meant to remove.
|
|
const text = ['- top level', ' - nested with [a link](gone.md)'].join('\n');
|
|
const f = checkInternalLinks({ files: { 'README.md': text }, present: ['README.md'] });
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), true);
|
|
});
|
|
|
|
test('a genuine indented code block is still skipped', () => {
|
|
const text = ['Example:', '', ' curl [x](not-real.md)'].join('\n');
|
|
const f = checkInternalLinks({ files: { 'README.md': text }, present: ['README.md'] });
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
// ------------------------------------------ badge honesty beyond shields.io
|
|
|
|
test('a self-hosted badge asserting a run is caught too', () => {
|
|
const f = checkBadges({ readme: '' });
|
|
assert.equal(f.some((x) => x.code === 'BADGE-STATIC-CLAIM'), true);
|
|
});
|
|
|
|
// ------------------------------------------------------------ install truth
|
|
|
|
test('the marketplace URL in the install block must be the real one', () => {
|
|
const readme = [
|
|
'## Install',
|
|
'claude plugin marketplace add https://git.fromaitochitta.com/open/WRONG.git',
|
|
`claude plugin install repo-standard@${MKT.name}`,
|
|
].join('\n');
|
|
const f = checkInstallBlock({ readme, name: 'repo-standard', klass: 'plugin' }, REGISTER);
|
|
assert.equal(f.some((x) => x.code === 'INSTALL-URL-MISMATCH' && x.bucket === 'broken'), true);
|
|
});
|
|
|
|
test('a plugin absent from the catalog has an install line that cannot work', () => {
|
|
// Syntax is not truth. The brief's first control asks whether the command
|
|
// works for a stranger, not whether it is well-formed.
|
|
const f = checkInstallTruth({ name: 'ghost', klass: 'plugin', catalogNames: ['repo-mailbox', 'llm-security'] });
|
|
assert.equal(f.some((x) => x.code === 'INSTALL-NOT-IN-CATALOG' && x.bucket === 'broken'), true);
|
|
});
|
|
|
|
test('a plugin present in the catalog passes install truth', () => {
|
|
const f = checkInstallTruth({ name: 'repo-mailbox', klass: 'plugin', catalogNames: ['repo-mailbox'] });
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('an unreachable catalog is SKIP, never a pass', () => {
|
|
const f = checkInstallTruth({ name: 'repo-mailbox', klass: 'plugin', catalogNames: null });
|
|
assert.equal(f[0].level, 'SKIP');
|
|
});
|
|
|
|
test('non-plugin classes are not measured against the catalog', () => {
|
|
const f = checkInstallTruth({ name: 'portfolio-optimiser', klass: 'standalone', catalogNames: null });
|
|
assert.equal(f.filter((x) => x.level !== 'OK').length, 0);
|
|
});
|
|
|
|
test('an indented code block stays code past its first line', () => {
|
|
// The blank-line rule fixed nested lists but broke multi-line indented blocks:
|
|
// only line 1 followed a blank line, so lines 2+ leaked back into scanning.
|
|
// Caught by the gate on this plugin's own SKILL.md, which shows a README
|
|
// template containing a link that does not exist relative to that file.
|
|
const text = [
|
|
'Template:',
|
|
'',
|
|
' # <name>',
|
|
' ## Changelog',
|
|
' See [CHANGELOG.md](CHANGELOG.md).',
|
|
].join('\n');
|
|
const f = checkInternalLinks({ files: { 'skills/x/SKILL.md': text }, present: ['skills/x/SKILL.md'] });
|
|
assert.equal(f.filter((x) => x.level === 'ERROR').length, 0);
|
|
});
|
|
|
|
test('an indented block ends at the first unindented line', () => {
|
|
const text = ['Template:', '', ' code here', '', 'Back to prose with [a link](gone.md).'].join('\n');
|
|
const f = checkInternalLinks({ files: { 'README.md': text }, present: ['README.md'] });
|
|
assert.equal(f.some((x) => x.code === 'LINK-INTERNAL-MISSING'), true);
|
|
});
|
|
|
|
// -------------------------------------------------------- fetchWithRetry
|
|
// fetchOrgListing/fetchCatalogNames are I/O shell, exercised live by the CLI —
|
|
// but the retry decision is pure once fetchImpl and sleep are injected, so it
|
|
// gets the same unit coverage as the classifiers. Measured 2026-08-04: 13
|
|
// script invocations in ~1s tripped an anonymous 429 with no retry at all;
|
|
// this is the fix, not a sweep tool (that belongs in org-ops).
|
|
function fakeResponse(status, headers = {}) {
|
|
return {
|
|
status,
|
|
ok: status >= 200 && status < 300,
|
|
headers: { get: (k) => headers[k.toLowerCase()] ?? null },
|
|
};
|
|
}
|
|
|
|
test('a 429 is retried once and a subsequent 200 is returned', async () => {
|
|
const calls = [];
|
|
const responses = [fakeResponse(429), fakeResponse(200)];
|
|
const fetchImpl = async () => responses.shift();
|
|
const sleep = async (ms) => calls.push(ms);
|
|
const res = await fetchWithRetry('https://x', {}, { fetchImpl, sleep });
|
|
assert.equal(res.status, 200);
|
|
assert.equal(calls.length, 1);
|
|
});
|
|
|
|
test('Retry-After (seconds) sets the wait, not the exponential default', async () => {
|
|
const calls = [];
|
|
const responses = [fakeResponse(429, { 'retry-after': '2' }), fakeResponse(200)];
|
|
const fetchImpl = async () => responses.shift();
|
|
const sleep = async (ms) => calls.push(ms);
|
|
await fetchWithRetry('https://x', {}, { fetchImpl, sleep });
|
|
assert.equal(calls[0], 2000);
|
|
});
|
|
|
|
test('no Retry-After header falls back to exponential backoff from baseDelayMs', async () => {
|
|
const calls = [];
|
|
const responses = [fakeResponse(429), fakeResponse(429), fakeResponse(200)];
|
|
const fetchImpl = async () => responses.shift();
|
|
const sleep = async (ms) => calls.push(ms);
|
|
await fetchWithRetry('https://x', {}, { fetchImpl, sleep, baseDelayMs: 1000 });
|
|
assert.deepEqual(calls, [1000, 2000]);
|
|
});
|
|
|
|
test('exponential backoff is capped, so a long retry budget does not wait minutes between attempts', async () => {
|
|
// Measured 2026-08-04 against the live forge: 20 parallel requests from one
|
|
// IP produced 429 with NO Retry-After header at all (nginx never sends one
|
|
// here) — the exponential fallback is the only path that ever runs in
|
|
// practice. Recovery was gradual, not a fixed-duration ban: a burst that
|
|
// size took up to ~15s to fully drain, and a 20s manual pause cleared it.
|
|
// Uncapped doubling would reach 32s on a single attempt; capping at 8s and
|
|
// extending the retry budget covers the measured recovery window without
|
|
// one attempt blocking for excessive time.
|
|
const calls = [];
|
|
const responses = [
|
|
fakeResponse(429),
|
|
fakeResponse(429),
|
|
fakeResponse(429),
|
|
fakeResponse(429),
|
|
fakeResponse(429),
|
|
fakeResponse(200),
|
|
];
|
|
const fetchImpl = async () => responses.shift();
|
|
const sleep = async (ms) => calls.push(ms);
|
|
await fetchWithRetry('https://x', {}, { fetchImpl, sleep, baseDelayMs: 1000, maxDelayMs: 8000, retries: 5 });
|
|
// Uncapped, the 5th delay would be 1000 * 2**4 = 16000.
|
|
assert.deepEqual(calls, [1000, 2000, 4000, 8000, 8000]);
|
|
});
|
|
|
|
test('retries are bounded — a persistent 429 returns the 429, not an infinite loop', async () => {
|
|
let calls = 0;
|
|
const fetchImpl = async () => fakeResponse(429);
|
|
const sleep = async () => {
|
|
calls += 1;
|
|
};
|
|
const res = await fetchWithRetry('https://x', {}, { fetchImpl, sleep, retries: 2, baseDelayMs: 1 });
|
|
assert.equal(res.status, 429);
|
|
assert.equal(calls, 2);
|
|
});
|
|
|
|
test('a non-429 response returns immediately — no retry, no sleep', async () => {
|
|
let fetchCalls = 0;
|
|
let sleepCalls = 0;
|
|
const fetchImpl = async () => {
|
|
fetchCalls += 1;
|
|
return fakeResponse(200);
|
|
};
|
|
const sleep = async () => {
|
|
sleepCalls += 1;
|
|
};
|
|
const res = await fetchWithRetry('https://x', {}, { fetchImpl, sleep });
|
|
assert.equal(res.status, 200);
|
|
assert.equal(fetchCalls, 1);
|
|
assert.equal(sleepCalls, 0);
|
|
});
|
|
|
|
// --------------------------------------------------- engine version stamp
|
|
// Measured 2026-08-04: the /repo-standard skill resolved to a cached 0.1.1
|
|
// plugin root while 0.2.0 was installed and the catalog pinned it. Two repos
|
|
// independently proved it by running both engines against the same checkout:
|
|
// 0.2.0 gave 0 WARN, 0.1.1 resurrected the three false positives fixed in
|
|
// v0.1.2/v0.1.3 — and nothing in the output said which engine had run. The
|
|
// header carrying no version is what let a stale engine look like a pass.
|
|
test('headerLine names the engine version, not just the repo status', () => {
|
|
const line = headerLine({ name: 'voyage', klass: 'plugin', traits: [], status: 'OK' }, '0.2.1');
|
|
assert.match(line, /voyage/);
|
|
assert.match(line, /OK/);
|
|
assert.match(line, /0\.2\.1/);
|
|
});
|
|
|
|
test('headerLine includes traits when present, same as the unversioned line did', () => {
|
|
const line = headerLine({ name: 'llm-security', klass: 'plugin', traits: ['security'], status: 'WARN' }, '0.2.1');
|
|
assert.match(line, /\{security\}/);
|
|
});
|
|
|
|
test('withEngineVersion adds the version without disturbing existing fields', () => {
|
|
const result = { name: 'x', klass: 'plugin', status: 'OK', findings: [] };
|
|
const stamped = withEngineVersion(result, '0.2.1');
|
|
assert.equal(stamped.engineVersion, '0.2.1');
|
|
assert.equal(stamped.name, 'x');
|
|
assert.equal(stamped.status, 'OK');
|
|
assert.deepEqual(result, { name: 'x', klass: 'plugin', status: 'OK', findings: [] });
|
|
});
|
|
|
|
// ---------------------------------------------------- engine commit stamp
|
|
// Measured 2026-08-09 by org-ops: a sweep stamped 18 raw files `0.4.0`, and
|
|
// four of them carried findings from a check that only exists in 0.5.0. The
|
|
// stamp lied without being broken — `9eb210b` (the feature) and `63c75d8` (the
|
|
// version bump) are TWO commits, so the worktree held new code under an old
|
|
// number for a window. The version names a FILE; only the sha names the CODE,
|
|
// and the bump is by definition a different commit from the change it describes.
|
|
|
|
test('headerLine names the engine commit alongside the version', () => {
|
|
const line = headerLine({ name: 'voyage', klass: 'plugin', traits: [], status: 'OK' }, '0.5.0', 'a7276e6f0d1e2b3c');
|
|
assert.match(line, /0\.5\.0/);
|
|
assert.match(line, /a7276e6/);
|
|
});
|
|
|
|
test('headerLine falls back to the version alone when the commit is underivable', () => {
|
|
const line = headerLine({ name: 'voyage', klass: 'plugin', traits: [], status: 'OK' }, '0.5.0', null);
|
|
assert.match(line, /0\.5\.0/);
|
|
assert.doesNotMatch(line, /null|undefined/);
|
|
});
|
|
|
|
test('withEngineVersion stamps the commit next to the version', () => {
|
|
const stamped = withEngineVersion({ name: 'x', status: 'OK', findings: [] }, '0.5.0', 'a7276e6f0d1e2b3c');
|
|
assert.equal(stamped.engineVersion, '0.5.0');
|
|
assert.equal(stamped.engineCommit, 'a7276e6f0d1e2b3c');
|
|
});
|
|
|
|
test('engineCommit is present-and-null when underivable, never absent', () => {
|
|
// An ABSENT key means an older engine; an explicit null means this engine ran
|
|
// and could not derive the sha. A consumer sorting raw files by stamp needs
|
|
// those two to be different, which is the whole point of the field.
|
|
const stamped = withEngineVersion({ name: 'x', status: 'OK', findings: [] }, '0.5.0', null);
|
|
assert.equal(Object.hasOwn(stamped, 'engineCommit'), true);
|
|
assert.equal(stamped.engineCommit, null);
|
|
});
|
|
|
|
test('readEngineCommit returns null outside a git checkout instead of throwing', () => {
|
|
// Derived from the engine's own checkout with no network call. A consumer of
|
|
// a tarball or a vendored copy has no HEAD, and that is not a crash.
|
|
assert.equal(readEngineCommit('/nonexistent-path-for-repo-standard-test'), null);
|
|
});
|