"""A read-only MCP server over one or more built collections (motor C21). It calls no model and writes no file. serve --bundle NAME WIKI [SOURCES] [--bundle NAME WIKI [SOURCES] ...] Speaks the MCP stdio transport: JSON-RPC 2.0, one message per line on stdin and stdout; anything else goes to stderr. Register it with Claude Code as, for example, `claude mcp add okf-wiki -- okf-wiki serve --bundle hage /path/wiki /path/sources`. Four tools, the skill's router for answering a question: collections each collection: its folders, pages, main pages, source files and okf_version search words against each page's title, other names and description in index.md (SPEC section 8) and its text (footnote lines and link targets left out); a page ranks by how rare its matching words are, a word in the title or in another name of the page (`x-aliases`, at the end of its index line as " (also: A; B)") counting twice; a word matches the same word or one with an ending of at most two letters on a stem of four (kmerge.word_match); main_only keeps main pages, and with no words lists them all, one per document page one concept by its id (SPEC section 2: its path without .md): frontmatter, other names, sources with quote and place, body, links out (each marked when it names no page) and the pages that link to it source lines of the frozen copy of a source around a cited line, with the heading above, each line's page, slide or cell, and the original file Titles and descriptions come from index.md, and page text, main pages and inbound links from a plain scan of the page files without parsing their YAML (which takes 67 s on wiki-38): the first call over wiki-38, 3 141 pages, reads it in 2.3 s (08.11), and a search after that takes milliseconds. A collection is read again when its index.md or its folder changes. Nothing outside a collection's folder or its copy of the sources is read. Plain file search in the same folders works beside it. """ from __future__ import annotations import argparse import json import math import re import sys from dataclasses import dataclass, field from pathlib import Path from typing import Any, TextIO from okfwiki.audit import LINK, SCHEME, read_document, resolve from okfwiki.common import place_words from okfwiki.kmerge import name_words, word_match from okfwiki.view import nearest_heading PROTOCOL_VERSIONS = ("2025-11-25", "2025-06-18", "2025-03-26", "2024-11-05") # newest first SERVER = {"name": "okf-wiki", "version": "1"} RESERVED = ("index.md", "log.md") # SPEC 3.1 FOOTNOTE_DEF = re.compile(r"^\[\^[^\]]+\]:") INDEX_LINE = re.compile(r"^[-*] \[(?P.*?)\]\((?P<href>[^()\s]+?\.md)\)(?: - (?P<desc>.*))?$") ALSO = re.compile(r"^(?P<desc>.*) \(also: (?P<names>.+)\)$") # a page's other names, as kassemble ends its index line DEFAULT_LIMIT = 20 DEFAULT_WINDOW = 2 # Claude Code (2.1.319, 08.11) kept a tool result of 47 062 characters whole and put one of 55 819 in a file; a page # longer than this comes in parts, each asked for with `part`. PART_CHARS = 40_000 INSTRUCTIONS = ( "Read-only lookups in built okf-wiki collections. No call here runs a model, and nothing is written.\n" "To answer a question: (1) `search` each collection's pages with the question's words, in the collection's own " "words and language: it matches words, not meaning, and says how many pages each word hits and how many pages " "matched of how many; (2) open the main page of the area with `page` (`search` with main_only lists the main " "pages, one per document); (3) follow its links_out with `page`, and linked_from to see what builds on it; " "(4) open the source at a footnote's line with `source` when the quote has to be checked or the page does not " "say enough. Cite the page and the source (document and place). Say plainly what the collections do not cover. " "Plain file search in the collection folders works beside this server.") class ToolError(Exception): """A call the server cannot answer: reported to the caller as a tool result with isError.""" @dataclass(frozen=True) class Collection: name: str wiki: Path sources: Path | None @dataclass class Entry: id: str title: str description: str title_words: set[str] # of the title and of the page's other names words: set[str] # title, other names, description and page text aliases: list[str] = field(default_factory=list) @dataclass class Catalogue: stamp: tuple[int, int, int] okf_version: str | None entries: list[Entry] main_for: dict[str, list[str]] linked_from: dict[str, list[str]] vocabulary: dict[str, set[int]] = field(default_factory=dict) # word -> entry indexes # --- reading a collection ---------------------------------------------------------------------------------------- def stamp(wiki: Path) -> tuple[int, int, int]: index = (wiki / "index.md").stat() return index.st_mtime_ns, index.st_size, wiki.stat().st_mtime_ns def frontmatter_text(text: str) -> tuple[str, str]: """(frontmatter, body) of a page as plain text, split at the closing --- (SPEC 4), without parsing the YAML.""" lines = text.split("\n") if not lines or lines[0].strip() != "---": return "", text end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None) if end is None: return "", text return "\n".join(lines[1:end]), "\n".join(lines[end + 1:]) def main_for_of(front: str) -> list[str]: """The sources a page is the main page of (`x-main-for`), read from the frontmatter text: a block list as kassemble writes it, or one value on the key's line.""" out: list[str] = [] lines = front.split("\n") for i, raw in enumerate(lines): if not raw.startswith("x-main-for:"): continue rest = raw[len("x-main-for:"):].strip() if rest: out.extend(v.strip().strip("'\"") for v in rest.strip("[]").split(",") if v.strip()) for item in lines[i + 1:]: if not item.startswith("- "): break out.append(item[2:].strip().strip("'\"")) return out def concept_id(rel: str) -> str: return rel[:-3] if rel.endswith(".md") else rel def load_catalogue(wiki: Path) -> Catalogue: st = stamp(wiki) index_text = (wiki / "index.md").read_text(encoding="utf-8") front, body = frontmatter_text(index_text) version = next((ln.split(":", 1)[1].strip().strip("'\"") for ln in front.split("\n") if ln.startswith("okf_version:")), None) entries: list[Entry] = [] seen: set[str] = set() for raw in body.split("\n"): m = INDEX_LINE.match(raw.strip()) if not m: continue rel = resolve("index.md", m.group("href")) if not rel or rel in seen or rel.split("/")[-1] in RESERVED: continue seen.add(rel) title, desc = m.group("title"), m.group("desc") or "" also = ALSO.match(desc) aliases = [a.strip() for a in also.group("names").split("; ") if a.strip()] if also else [] desc = also.group("desc") if also else desc tw = set(name_words(" ".join([title, *aliases]))) entries.append(Entry(concept_id(rel), title, desc, tw, tw | set(name_words(desc)), aliases)) main_for: dict[str, list[str]] = {} linked: dict[str, set[str]] = {} body_words: dict[str, set[str]] = {} for p in sorted(wiki.rglob("*.md")): rel = p.relative_to(wiki).as_posix() if p.name in RESERVED or not p.is_file(): continue try: page_front, page_body = frontmatter_text(p.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError): continue mains = main_for_of(page_front) if mains: main_for[concept_id(rel)] = mains text = "\n".join(ln for ln in page_body.split("\n") if not FOOTNOTE_DEF.match(ln)) body_words[concept_id(rel)] = set(name_words(LINK.sub(lambda m: m.group(1), text))) for m in LINK.finditer(page_body): target = m.group(2) if not target or target.startswith("#") or SCHEME.match(target): continue to = resolve(rel, target) if to and to.endswith(".md") and to != rel: linked.setdefault(concept_id(to), set()).add(concept_id(rel)) cat = Catalogue(st, version, entries, main_for, {k: sorted(v) for k, v in linked.items()}) for n, e in enumerate(entries): e.words |= body_words.get(e.id, set()) for w in e.words: cat.vocabulary.setdefault(w, set()).add(n) return cat # --- the server -------------------------------------------------------------------------------------------------- def schema(props: dict[str, Any], required: list[str]) -> dict[str, Any]: return {"type": "object", "properties": props, "required": required, "additionalProperties": False} COLLECTION_ARG = {"type": "string", "description": "The collection's name, as `collections` lists it."} TOOLS: list[dict[str, Any]] = [ {"name": "collections", "description": "List the collections this server reads: name, folders, pages, main pages, source files.", "inputSchema": schema({}, [])}, {"name": "search", "description": ("Search the pages of one or all collections (title, other names, description and text) for words. " "Matches words, not meaning: use the collection's own words and language, and ask again with " "other words when the counts say a word hits nothing. main_only keeps main pages (one per " "document); main_only with no words lists them all."), "inputSchema": schema({ "words": {"type": "string", "description": "The words to look for, separated by spaces."}, "collection": COLLECTION_ARG, "main_only": {"type": "boolean", "description": "Only main pages (default false)."}, "limit": {"type": "integer", "minimum": 1, "description": f"Results per collection (default {DEFAULT_LIMIT})."}, "offset": {"type": "integer", "minimum": 0, "description": "Results to skip, for the next page (default 0)."}, }, [])}, {"name": "page", "description": ("One concept page by its id (its file name without .md): title, description, sources with the " "verbatim quote and its place, the body with footnotes, links out and the pages linking to it. " "A long page comes in parts: `parts` says how many, ask for the next with `part`."), "inputSchema": schema({"collection": COLLECTION_ARG, "id": {"type": "string", "description": "The concept id, e.g. from search."}, "part": {"type": "integer", "minimum": 1, "description": "Which part (default 1)."}}, ["collection", "id"])}, {"name": "source", "description": ("Lines of a source's frozen copy around a cited line, with the heading above, each line's page, " "slide or cell, and the original file. resource and line are a page source's resource and line."), "inputSchema": schema({ "collection": COLLECTION_ARG, "resource": {"type": "string", "description": "The source's resource, e.g. /kilde/report.pdf.md."}, "line": {"type": "integer", "minimum": 1, "description": "The cited line (1-based)."}, "before": {"type": "integer", "minimum": 0, "description": f"Lines before it (default {DEFAULT_WINDOW})."}, "after": {"type": "integer", "minimum": 0, "description": f"Lines after it (default {DEFAULT_WINDOW})."}, }, ["collection", "resource", "line"])}, ] for _tool in TOOLS: _tool["annotations"] = {"readOnlyHint": True, "destructiveHint": False, "idempotentHint": True, "openWorldHint": False} def int_arg(args: dict[str, Any], key: str, default: int, least: int) -> int: value = args.get(key, default) if type(value) is not int or value < least: raise ToolError(f"{key} must be a whole number of at least {least}") return int(value) class Server: def __init__(self, collections: list[Collection]) -> None: self.collections = {c.name: c for c in collections} self.catalogues: dict[str, Catalogue] = {} # protocol def handle(self, msg: Any) -> dict[str, Any] | None: if not isinstance(msg, dict) or msg.get("jsonrpc") != "2.0" or not isinstance(msg.get("method"), str): return error(msg.get("id") if isinstance(msg, dict) else None, -32600, "not a JSON-RPC 2.0 request") if "id" not in msg: return None # a notification mid, method, params = msg["id"], msg["method"], msg.get("params") or {} if method == "initialize": asked = params.get("protocolVersion") if isinstance(params, dict) else None version = asked if asked in PROTOCOL_VERSIONS else PROTOCOL_VERSIONS[0] return reply(mid, {"protocolVersion": version, "capabilities": {"tools": {"listChanged": False}}, "serverInfo": SERVER, "instructions": INSTRUCTIONS}) if method == "ping": return reply(mid, {}) if method == "tools/list": return reply(mid, {"tools": TOOLS}) if method == "tools/call": name = params.get("name") if isinstance(params, dict) else None args = params.get("arguments") or {} if isinstance(params, dict) else {} tool = getattr(self, f"tool_{name}", None) if name in {t["name"] for t in TOOLS} else None if tool is None or not isinstance(args, dict): return error(mid, -32602, f"unknown tool {name!r}" if tool is None else "arguments must be an object") try: data, is_error = tool(args), False except ToolError as exc: data, is_error = {"error": str(exc)}, True text = dumps(data) return reply(mid, {"content": [{"type": "text", "text": text}], "isError": is_error}) return error(mid, -32601, f"method not found: {method}") # collections def collection(self, name: Any) -> Collection: if name not in self.collections: raise ToolError(f"no collection {name!r}; this server reads {', '.join(sorted(self.collections))}") return self.collections[str(name)] def catalogue(self, c: Collection) -> Catalogue: try: current = stamp(c.wiki) cat = self.catalogues.get(c.name) if cat is None or cat.stamp != current: cat = self.catalogues[c.name] = load_catalogue(c.wiki) except (OSError, UnicodeDecodeError) as exc: raise ToolError(f"collection {c.name} cannot be read: {exc}") from exc return cat def tool_collections(self, args: dict[str, Any]) -> dict[str, Any]: out = [] for c in self.collections.values(): cat = self.catalogue(c) files = None if c.sources is not None: files = sum(1 for p in (c.sources / "kilde").rglob("*") if p.is_file()) out.append({"collection": c.name, "wiki": str(c.wiki.resolve()), "sources": str(c.sources.resolve()) if c.sources else None, "pages": len(cat.entries), "main_pages": len(cat.main_for), "source_files": files, "okf_version": cat.okf_version}) return {"collections": out} # search def tool_search(self, args: dict[str, Any]) -> dict[str, Any]: words = args.get("words", "") if not isinstance(words, str): raise ToolError("words must be a string") main_only = args.get("main_only", False) is True limit = int_arg(args, "limit", DEFAULT_LIMIT, 1) offset = int_arg(args, "offset", 0, 0) asked = list(dict.fromkeys(name_words(words))) if not asked and not main_only: raise ToolError("give words to search for, or main_only to list the main pages") chosen = [self.collection(args["collection"])] if "collection" in args else list(self.collections.values()) return {"collections": [self.search_one(c, asked, main_only, limit, offset) for c in chosen]} def search_one(self, c: Collection, asked: list[str], main_only: bool, limit: int, offset: int) -> dict[str, Any]: cat = self.catalogue(c) n = len(cat.entries) hits: dict[int, list[str]] = {} counts = [] weight: dict[str, float] = {} for word in asked: found: set[int] = set() for known, at in cat.vocabulary.items(): if word_match(word, known): found |= at counts.append({"word": word, "pages": len(found)}) weight[word] = math.log((n + 1) / (len(found) + 0.5)) for i in found: hits.setdefault(i, []).append(word) if asked: pool = list(hits) else: pool = list(range(n)) results = [] for i in pool: e = cat.entries[i] if main_only and e.id not in cat.main_for: continue matched = hits.get(i, []) score = sum(weight[w] * (2 if any(word_match(w, t) for t in e.title_words) else 1) for w in matched) results.append((-score, e.title.casefold(), e.id, e, matched, score)) results.sort(key=lambda r: (r[0], r[1], r[2])) page = results[offset:offset + limit] return {"collection": c.name, "pages": n, "matched": len(results), "words": counts, "offset": offset, "more": offset + limit < len(results), "results": [{"id": e.id, "title": e.title, "aliases": e.aliases, "description": e.description, "main_for": cat.main_for.get(e.id, []), "score": round(score, 3), "words": matched} for _, _, _, e, matched, score in page]} # page def page_rel(self, c: Collection, pid: Any) -> str: if not isinstance(pid, str) or not pid.strip(): raise ToolError("id must be a concept id, e.g. from search") rel = pid.strip() rel = rel if rel.endswith(".md") else rel + ".md" norm = resolve("index.md", rel.lstrip("/")) if not rel.startswith("/") else None path = (c.wiki / norm) if norm else None if (not norm or norm.split("/")[-1] in RESERVED or path is None or not path.is_file() or not path.resolve().is_relative_to(c.wiki.resolve())): raise ToolError(f"no concept {pid!r} in {c.name}; find its id with search") return norm def tool_page(self, args: dict[str, Any]) -> dict[str, Any]: c = self.collection(args.get("collection")) cat = self.catalogue(c) rel = self.page_rel(c, args.get("id")) try: doc = read_document(c.wiki / rel, rel) except (OSError, UnicodeDecodeError) as exc: raise ToolError(f"{rel} cannot be read: {exc}") from exc fm = doc.frontmatter or {} srcs = fm.get("sources", []) sources = [] for s in srcs if isinstance(srcs, list) else [srcs]: if not isinstance(s, dict): continue raw_loc = s.get("x-locator") loc: dict[str, Any] = raw_loc if isinstance(raw_loc, dict) else {} sources.append({"id": s.get("id"), "resource": s.get("resource"), "title": s.get("title"), "line": loc.get("line"), "place": place_words(loc) if loc else None, "quote": s.get("x-quote"), "provenance": s.get("x-provenance")}) out_links: list[dict[str, Any]] = [] seen: set[str] = set() for m in LINK.finditer(doc.body): target = m.group(2) if not target or target.startswith("#") or SCHEME.match(target): continue to = resolve(rel, target) if not to or not to.endswith(".md") or to == rel or to in seen: continue seen.add(to) out_links.append({"id": concept_id(to), "exists": (c.wiki / to).is_file()}) mains = fm.get("x-main-for", []) names = fm.get("x-aliases", []) head = {"collection": c.name, "id": concept_id(rel), "path": str((c.wiki / rel).resolve()), "title": fm.get("title"), "aliases": names if isinstance(names, list) else [names], "description": fm.get("description"), "type": fm.get("type"), "main_for": mains if isinstance(mains, list) else [mains], "links_out": out_links, "linked_from": cat.linked_from.get(concept_id(rel), [])} parts = split_parts(head, doc.body.strip(), sources) part = int_arg(args, "part", 1, 1) if part > len(parts): raise ToolError(f"{concept_id(rel)} has {len(parts)} part(s), not {part}") return parts[part - 1] # source def tool_source(self, args: dict[str, Any]) -> dict[str, Any]: c = self.collection(args.get("collection")) if c.sources is None: raise ToolError(f"collection {c.name} was started without its sources folder; give it as the third " f"value of --bundle") res = args.get("resource") line = int_arg(args, "line", 0, 1) before = int_arg(args, "before", DEFAULT_WINDOW, 0) after = int_arg(args, "after", DEFAULT_WINDOW, 0) rel = resolve("index.md", res) if isinstance(res, str) and res and not SCHEME.match(res) else None root = c.sources.resolve() path = (c.sources / rel) if rel else None if (not rel or not rel.startswith("kilde/") or path is None or not path.is_file() or not path.resolve().is_relative_to(root)): raise ToolError(f"no source {res!r} in the sources of {c.name}; use a resource a page's sources name") try: lines = path.read_text(encoding="utf-8").splitlines() except (OSError, UnicodeDecodeError) as exc: raise ToolError(f"{rel} cannot be read: {exc}") from exc if line > len(lines): raise ToolError(f"{res} has {len(lines)} lines, not {line}") entry = self.manifest_entry(c, rel) places = entry.get("places") if isinstance(entry.get("places"), list) else None shown = [] for n in range(max(1, line - before), min(len(lines), line + after) + 1): item: dict[str, Any] = {"line": n, "text": lines[n - 1]} place = places[n - 1] if places is not None and n - 1 < len(places) else None if isinstance(place, dict): item["place"] = place_words(place) shown.append(item) heading = nearest_heading(lines, line) out: dict[str, Any] = {"collection": c.name, "resource": res, "path": str(path.resolve()), "lines_total": len(lines), "lines": shown, "heading": {"line": heading[0], "text": heading[1]} if heading else None} if isinstance(entry.get("original"), str): original: dict[str, Any] = {"name": Path(entry["original"]).name} if isinstance(entry.get("url"), str): original["url"] = entry["url"] copy = entry.get("original_copy") if isinstance(copy, str) and (c.sources / copy).resolve().is_relative_to(root): original["copy"] = str((c.sources / copy).resolve()) out["original"] = original elif isinstance(entry.get("url"), str): out["original"] = {"url": entry["url"]} return out def manifest_entry(self, c: Collection, rel: str) -> dict[str, Any]: assert c.sources is not None try: doc = json.loads((c.sources / "source-manifest.json").read_text(encoding="utf-8")) except (OSError, ValueError): return {} for e in doc.get("sources", []) if isinstance(doc, dict) else []: if isinstance(e, dict) and e.get("path") == rel: return e return {} def dumps(data: Any) -> str: return json.dumps(data, ensure_ascii=False, separators=(",", ":")) def split_parts(head: dict[str, Any], body: str, sources: list[dict[str, Any]]) -> list[dict[str, Any]]: """A page as parts of at most PART_CHARS characters of JSON each: the first carries the page's fields, every part its share of the body (whole paragraphs, in order) and then of the sources (whole entries, in order). The bodies joined with a blank line are the page's body, and nothing is left out; one paragraph or entry longer than a part stands alone in its part.""" items: list[tuple[str, Any]] = [("p", p) for p in body.split("\n\n")] if body else [] items += [("s", s) for s in sources] def build(base: dict[str, Any], chunk: list[tuple[str, Any]], k: int, n: int) -> dict[str, Any]: return {**base, "part": k, "parts": n, "body": "\n\n".join(v for t, v in chunk if t == "p"), "sources": [v for t, v in chunk if t == "s"]} chunks: list[list[tuple[str, Any]]] = [[]] for item in items: base = head if len(chunks) == 1 else {"collection": head["collection"], "id": head["id"]} trial = chunks[-1] + [item] if chunks[-1] and len(dumps(build(base, trial, 99999, 99999))) > PART_CHARS: chunks.append([item]) else: chunks[-1] = trial n = len(chunks) return [build(head if k == 1 else {"collection": head["collection"], "id": head["id"]}, chunk, k, n) for k, chunk in enumerate(chunks, 1)] def reply(mid: Any, result: dict[str, Any]) -> dict[str, Any]: return {"jsonrpc": "2.0", "id": mid, "result": result} def error(mid: Any, code: int, message: str) -> dict[str, Any]: return {"jsonrpc": "2.0", "id": mid, "error": {"code": code, "message": message}} def run(server: Server, inp: TextIO, out: TextIO) -> None: for raw in inp: if not raw.strip(): continue try: msg = json.loads(raw) except ValueError: answer: dict[str, Any] | None = error(None, -32700, "not JSON") else: answer = server.handle(msg) if answer is not None: out.write(json.dumps(answer, ensure_ascii=False) + "\n") out.flush() def collections_from(groups: list[list[str]]) -> list[Collection]: out: list[Collection] = [] for g in groups: if len(g) not in (2, 3): raise ValueError(f"--bundle takes NAME WIKI [SOURCES], not {' '.join(g)}") name, wiki = g[0], Path(g[1]).expanduser() sources = Path(g[2]).expanduser() if len(g) == 3 else None if any(c.name == name for c in out): raise ValueError(f"two collections are named {name}") if not (wiki / "index.md").is_file(): raise ValueError(f"{wiki} has no index.md, so it is not a built collection") if sources is not None and not (sources / "source-manifest.json").is_file(): raise ValueError(f"{sources} has no source-manifest.json, so it is not a sources folder") out.append(Collection(name, wiki, sources)) return out def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(prog="okf-wiki serve", description=__doc__.split("\n")[0]) ap.add_argument("--bundle", action="append", nargs="+", required=True, metavar="NAME WIKI [SOURCES]", help="a collection to serve: its name, its folder, and its sources folder (repeat for more)") try: ns = ap.parse_args(argv) except SystemExit as exc: return int(exc.code or 0) if isinstance(exc.code, int) else 2 try: collections = collections_from(ns.bundle) except ValueError as exc: print(f"serve: {exc}", file=sys.stderr) return 2 print(f"okf-wiki serve: {', '.join(c.name for c in collections)}", file=sys.stderr) run(Server(collections), sys.stdin, sys.stdout) return 0 if __name__ == "__main__": sys.exit(main())