okf-wiki 1.1.1

First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-11-14 21:03:20 +01:00
commit 0fd86a23a3
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
227 changed files with 96077 additions and 0 deletions

1
okfwiki/__init__.py Normal file
View file

@ -0,0 +1 @@
"""okf-wiki: tools that check and render an OKF wiki. No tool here calls a language model (CLAUDE.md rule 1)."""

969
okfwiki/audit.py Normal file
View file

@ -0,0 +1,969 @@
"""The auditor: checks an OKF wiki bundle against its raw sources. It never calls a language model (CLAUDE.md rule 1).
Usage:
python3 -m okfwiki.audit BUNDLE SOURCES
BUNDLE is the wiki directory. SOURCES is the raw-source root that a bundle-absolute `sources[].resource`
(`/kilde/x.md`) resolves against, and that holds the source manifest `source-manifest.json` (contract K1). L1a reads
raw sources as UTF-8 text with a `{ line: N }` locator only.
Output on stdout, one JSON object: {"findings": [{"check", "group", "file", "line", "message"}, ...],
"summary": {<group>: n}, "link_rule": {"line", "main_pages": {"pass", "of"}, "linked": {"pass", "of"}, "misplaced",
"broken", "not_to_concept", "max_out_degree", "provenance"}}. `link_rule` is criterion 6 as numbers (contract K2
point 7): "main pages: N of N sources · linked from a main page: N of N pages".
Exit 0 clean, 1 at least one finding, 2 the audit could not run (bad arguments; no valid K1 file at SOURCES, since the
main-page rule cannot run without it, K2 point 6; or no main-topic source in K1 and no concept page, no denominator).
Four groups are reported apart, because they answer different questions:
- spec-11: OKF v0.2 SPEC section 11 conformance (PLAN.md criterion 5).
- links: okf-wiki's own link rules. SPEC 6.1 says consumers MUST tolerate broken links, so these are not section 11.
- citations: every body sentence carries a footnote, every footnote a source, every source a verbatim quote at its
locator (PLAN.md criterion 1, format choice f).
- format: format choices (a), (b), (c) and (e) in PLAN.md section Rammer.
The exact definitions of the link classes, including link-misplaced (b) of criterion 6 and the main-page rule
(contract K2: main-page-missing, main-page-duplicate, main-page-invalid, no-main-page-link), are in
gates/mutants.manifest.json under provenance.link_rules. The rule of 04.10.2026 (an inbound link from any other concept
page) is no finding any more; link_rule_report() still counts it, for the linker and the trial.
The function audit() reports the main-page rule only when SOURCES holds a K1 file (K2 point 18); without one it gives
the other checks, as the trial's tests and the assembler's call it on roots without K1.
"""
from __future__ import annotations
import json
import os
import re
import sys
from dataclasses import asdict, dataclass, field
from datetime import date, datetime
from pathlib import Path, PurePosixPath
from typing import Any, Callable
from urllib.parse import unquote
import yaml
from okfwiki.common import place_fits
RESERVED = ("index.md", "log.md") # SPEC 3.1
GROUPS = ("spec-11", "links", "citations", "format")
GROUP_OF = {
"spec11-1-frontmatter": "spec-11",
"spec11-2-type": "spec-11",
"spec11-3-index": "spec-11",
"spec11-3-log": "spec-11",
"broken-link": "links",
"orphan-page": "links",
"index-coverage": "links",
"link-misplaced": "links",
"main-page-missing": "links",
"main-page-duplicate": "links",
"main-page-invalid": "links",
"no-main-page-link": "links",
"uncited-sentence": "citations",
"quote-not-verbatim": "citations",
"quote-wrong-locator": "citations",
"quote-wrong-place": "citations",
"quote-missing": "citations",
"locator-unsupported": "citations",
"source-unreadable": "citations",
"source-missing-id": "citations",
"source-duplicate-id": "citations",
"footnote-unknown-id": "citations",
"resource-not-bundle-absolute": "format",
"body-link-bundle-absolute": "format",
"timestamp-form": "format",
"verified-not-human": "format",
}
class _Loader(yaml.SafeLoader):
"""SafeLoader without the timestamp resolver, as Google's reference `document.py` does (format choice c)."""
_Loader.yaml_implicit_resolvers = {
ch: [(tag, rx) for tag, rx in resolvers if tag != "tag:yaml.org,2002:timestamp"]
for ch, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items()
}
@dataclass(frozen=True)
class Finding:
check: str
group: str
file: str
line: int | None
message: str
def finding(check: str, file: str, line: int | None, message: str) -> Finding:
return Finding(check=check, group=GROUP_OF[check], file=file, line=line, message=message)
# --- documents -------------------------------------------------------------------------------------------------
@dataclass
class Document:
"""One markdown file split as SPEC section 4 says: a frontmatter block, then the body."""
rel: str
has_frontmatter: bool
frontmatter: dict[str, Any] | None # None when absent or not parseable to a mapping
frontmatter_error: str
body: str
body_start: int # 1-based file line of the body's first line
def read_document(path: Path, rel: str) -> Document:
text = path.read_text(encoding="utf-8")
lines = text.split("\n")
if not lines or lines[0].strip() != "---":
return Document(rel, False, None, "", text, 1)
end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
if end is None:
return Document(rel, True, None, "the frontmatter block is not closed with ---", "", len(lines) + 1)
body = "\n".join(lines[end + 1:])
try:
data = yaml.load("\n".join(lines[1:end]), Loader=_Loader)
except yaml.YAMLError as exc:
first = str(exc).splitlines()[0] if str(exc) else "YAML error"
return Document(rel, True, None, f"the frontmatter does not parse: {first}", body, end + 2)
if data is None:
data = {}
if not isinstance(data, dict):
return Document(rel, True, None, "the frontmatter is not a YAML mapping", body, end + 2)
return Document(rel, True, data, "", body, end + 2)
# --- markdown body ---------------------------------------------------------------------------------------------
# A code span: a backtick run not escaped and not part of a longer run, closed by a run of the same length (6.1).
CODE_SPAN = re.compile(r"(?<![`\\])(`+)(?!`)(.+?)(?<!`)\1(?!`)", re.S)
FENCE = re.compile(r"^(?:[ \t]*>)*[ \t]*(`{3,}|~{3,})") # in a list item or a quote block too
LINK = re.compile(r"(?<![!\\])\[([^\]\n]*)\]\(\s*<?([^()\s<>]*)>?(?:\s+\"[^\"]*\")?\s*\)")
FOOTNOTE_REF = re.compile(r"\[\^([^\]\s]+)\](?!:)")
FOOTNOTE_DEF = re.compile(r"^\[\^([^\]\s]+)\]:")
HEADING = re.compile(r"^ {0,3}(#{1,6})(?:\s+(.*?))?\s*#*\s*$")
LIST_ITEM = re.compile(r"^\s*(?:[-*+]|\d+[.)])\s+")
SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:")
# A sentence ends at . ! or ?, optionally followed by closing quotes or brackets and by footnote references,
# then whitespace. sentences() skips an end followed by a lowercase letter, which keeps abbreviations such as
# "e.g. overnight" inside their sentence.
SENTENCE_END = re.compile(r"[.!?][\"'»”’)\]]*(?:\[\^[^\]\s]+\])*(?=\s+(\S))")
# A period after one of these ends no sentence, whatever follows ("kl. 14", "f.eks. Oslo", "e.g. Claude Code"). Only
# abbreviations that seldom end a sentence are here: "osv.", "etc." and "m.m." usually close one and still cut. A
# single capital letter before the period is an initial and does not cut either.
ABBREVIATIONS = frozenset(
"f.eks bl.a kl ca nr jf jfr dvs evt ev inkl ekskl mht pga ift iht vedr ang tlf stk pkt kap avd ref hhv mill mrd"
" e.g i.e cf vs approx incl mr mrs ms dr prof".split())
ABBREVIATION_BEFORE = re.compile(r"([^\s(\[\"'«]+)\.$")
LINK_ONLY = re.compile(r"^(?:\s*\[[^\]\n]*\]\([^()\s]*\)\s*[,;]?)+\s*$")
def fence_marker(raw: str) -> re.Match[str] | None:
"""The fence run that opens or closes a fenced block on this line (CommonMark 4.5), or None. A backtick run with a
backtick after it is an inline code span, not a fence."""
f = FENCE.match(raw)
if f and f.group(1)[0] == "`" and "`" in raw[f.end():]:
return None
return f
def closes_fence(f: re.Match[str] | None, raw: str, fence: str) -> bool:
"""A fence closes on a run of the opening character at least as long as the opening run, with nothing after it."""
return bool(f and f.group(1)[0] == fence[0] and len(f.group(1)) >= len(fence) and not raw[f.end():].strip())
def mask_code(text: str) -> str:
"""Blank out code so nothing inside it reads as a link, footnote or sentence end; keeps offsets and line breaks.
A fenced block (backticks or tildes) is masked line by line after its opening run. An inline code span pairs
backtick runs of equal length only inside one block: a blank line, a heading, a list item or a footnote
definition starts a new one, and a backslash before a backtick makes it literal (CommonMark 6.1)."""
out: list[str] = []
block: list[str] = []
def flush() -> None:
if block:
out.append(CODE_SPAN.sub(lambda m: m.group(1) + "x" * len(m.group(2)) + m.group(1), "\n".join(block)))
block.clear()
fence = ""
for raw in text.split("\n"):
f = fence_marker(raw)
if fence:
if closes_fence(f, raw, fence):
fence = ""
out.append(raw)
else:
out.append("x" * len(raw))
elif f:
flush()
fence = f.group(1)
out.append(raw[:f.end()] + "x" * len(raw[f.end():]))
elif not raw.strip() or HEADING.match(raw):
flush()
out.append(CODE_SPAN.sub(lambda m: m.group(1) + "x" * len(m.group(2)) + m.group(1), raw))
else:
if LIST_ITEM.match(raw) or FOOTNOTE_DEF.match(raw.strip()):
flush()
block.append(raw)
flush()
return "\n".join(out)
def link_matches(text: str, masked: str | None = None) -> list[re.Match[str]]:
"""The inline links of text. A [ inside a code span opens no link (CommonMark 6.1): a match that opens in code
and closes outside it is passed over from its bracket, so the link after it is found. A link written wholly
inside code is kept, and placed_links calls it code. masked is mask_code(text), or the same slice of it."""
masked = mask_code(text) if masked is None else masked
out: list[re.Match[str]] = []
pos = 0
while (m := LINK.search(text, pos)) is not None:
if masked[m.start()] != text[m.start()] and masked[m.end() - 1] == text[m.end() - 1]:
pos = m.start() + 1
continue
out.append(m)
pos = m.end()
return out
def sub_links(repl: Callable[[re.Match[str]], str], text: str) -> str:
"""LINK.sub over link_matches: a bracket inside a code span is left as it stands."""
parts: list[str] = []
last = 0
for m in link_matches(text):
parts += [text[last:m.start()], repl(m)]
last = m.end()
return "".join(parts) + text[last:]
@dataclass
class Block:
line: int
text: str
is_list_item: bool
def prose_blocks(doc: Document) -> list[Block]:
"""Paragraphs and list items of the body; headings and footnote definitions are not prose."""
blocks: list[Block] = []
current: Block | None = None
for i, raw in enumerate(doc.body.split("\n")):
line_no = doc.body_start + i
if not raw.strip():
current = None
continue
if HEADING.match(raw) or FOOTNOTE_DEF.match(raw.strip()):
current = None
continue
if LIST_ITEM.match(raw):
current = Block(line_no, LIST_ITEM.sub("", raw, count=1), True)
blocks.append(current)
elif current is None:
current = Block(line_no, raw.strip(), False)
blocks.append(current)
else:
current.text += " " + raw.strip()
return blocks
def sentence_spans(text: str) -> list[tuple[int, int]]:
"""(start, end) of every sentence in text, found on the code-masked text, so the offsets fit the original too.
A bundle link never moves a cut: the ends are found on the masked text with every bundle link outside code
reduced to its anchor text (klink.reduce_links' notion), and each cut is mapped back to the linked text, after
the whole link when it falls at the anchor's end. So "U.S. [primary](p.md) was" stays one sentence, as "U.S.
primary was" is. Text without a bundle link is split exactly as before: the reduced text is the masked text."""
masked = mask_code(text)
plain, after = masked, list(range(1, len(masked) + 1)) # after[i]: offset in masked just after plain[i]
links = [m for m in link_matches(text, masked) if masked[m.start()] == "[" and is_bundle_link(m.group(2))]
if links:
parts: list[str] = []
after = []
last = 0
for m in links:
parts += [masked[last:m.start()], masked[m.start(1):m.end(1)]]
after += list(range(last + 1, m.start() + 1)) + list(range(m.start(1) + 1, m.end(1) + 1))
if m.end(1) > m.start(1):
after[-1] = m.end()
last = m.end()
parts.append(masked[last:])
after += list(range(last + 1, len(masked) + 1))
plain = "".join(parts)
cuts: list[tuple[int, int]] = []
start = 0
for e in sentence_ends(plain):
end = after[e - 1]
cuts.append((start, end))
start = end
cuts.append((start, len(masked)))
out: list[tuple[int, int]] = []
for a, b in cuts:
piece = masked[a:b]
if not re.search(r"\w", FOOTNOTE_REF.sub("", piece)):
continue
lead = len(piece) - len(piece.lstrip())
out.append((a + lead, a + len(piece.rstrip())))
return out
def sentence_ends(masked: str) -> list[int]:
"""The offset just after each sentence end of a code-masked text."""
ends: list[int] = []
for m in SENTENCE_END.finditer(masked):
if m.group(1).islower():
continue
word = ABBREVIATION_BEFORE.search(masked, max(0, m.start() - 20), m.start() + 1)
w = word.group(1) if word else ""
if masked[m.start()] == "." and (w.lower() in ABBREVIATIONS and w in (w.lower(), w.capitalize())
or len(w) == 1 and w.isupper()):
continue # an abbreviation, or an initial ("Steven L. Rickman")
ends.append(m.end())
return ends
def sentences(text: str) -> list[str]:
masked = mask_code(text)
return [masked[a:b] for a, b in sentence_spans(text)]
def body_links(doc: Document) -> list[tuple[int, str]]:
"""(file line, target) for every inline markdown link in the body, outside code spans."""
out: list[tuple[int, str]] = []
for i, raw in masked_lines(doc.body):
for m in LINK.finditer(raw):
out.append((doc.body_start + i, m.group(2)))
return out
def masked_lines(body: str) -> list[tuple[int, str]]:
"""(index, masked line) per line of the unmasked body. mask_code also masks a fenced block's line breaks, so
the masked text is cut at the body's own line offsets: a line after a fenced block keeps its own number."""
masked = mask_code(body)
out: list[tuple[int, str]] = []
offset = 0
for i, raw in enumerate(body.split("\n")):
out.append((i, masked[offset:offset + len(raw)]))
offset += len(raw) + 1
return out
def is_bundle_link(target: str) -> bool:
return bool(target) and not target.startswith("#") and not SCHEME.match(target)
def resolve(rel_file: str, target: str) -> str | None:
"""Bundle-relative POSIX path a link or path names, or None if it leaves the bundle (SPEC 6.1, 6.2)."""
path = unquote(target.split("#", 1)[0].split("?", 1)[0])
if path.startswith("/"):
joined = path.lstrip("/")
else:
joined = str(PurePosixPath(rel_file).parent / path)
norm = os.path.normpath(joined).replace(os.sep, "/")
if norm == ".":
return ""
if norm == ".." or norm.startswith("../"):
return None
return norm
# --- the audit -------------------------------------------------------------------------------------------------
@dataclass
class Bundle:
root: Path
sources: Path
docs: dict[str, Document] = field(default_factory=dict)
_places: dict[str, list[Any]] | None = None
def places(self, rel: str) -> list[Any] | None:
"""The place in the original of each line of the raw source at rel (K1 `places`, PDF and Office), or None."""
if self._places is None:
self._places = {}
try:
doc = json.loads((self.sources / "source-manifest.json").read_text(encoding="utf-8"))
for e in doc.get("sources", []) if isinstance(doc, dict) else []:
if isinstance(e, dict) and isinstance(e.get("path"), str) and isinstance(e.get("places"), list):
self._places[e["path"]] = e["places"]
except (OSError, ValueError):
pass
return self._places.get(rel)
@property
def concepts(self) -> list[Document]:
return [d for rel, d in sorted(self.docs.items()) if PurePosixPath(rel).name not in RESERVED]
def named(self, name: str) -> list[Document]:
return [d for rel, d in sorted(self.docs.items()) if PurePosixPath(rel).name == name]
def is_file(self, rel: str | None) -> bool:
return rel is not None and rel != "" and (self.root / rel).is_file()
def load_bundle(root: Path, sources: Path) -> Bundle:
b = Bundle(root, sources)
for p in sorted(root.rglob("*.md")):
if p.is_file():
rel = p.relative_to(root).as_posix()
b.docs[rel] = read_document(p, rel)
return b
def check_spec11_concepts(b: Bundle) -> list[Finding]:
"""SPEC 11.1 and 11.2 on every non-reserved .md file in the tree."""
out: list[Finding] = []
for d in b.concepts:
if not d.has_frontmatter:
out.append(finding("spec11-1-frontmatter", d.rel, 1, "no YAML frontmatter block (SPEC 11.1)"))
elif d.frontmatter is None:
out.append(finding("spec11-1-frontmatter", d.rel, 1, d.frontmatter_error + " (SPEC 11.1)"))
else:
t = d.frontmatter.get("type")
if not isinstance(t, str) or not t.strip():
out.append(finding("spec11-2-type", d.rel, 1, "no non-empty type field (SPEC 11.2)"))
return out
def body_headings(doc: Document) -> list[tuple[int, int, str]]:
"""(file line, level, text) for every ATX heading in the body."""
out: list[tuple[int, int, str]] = []
for i, raw in enumerate(doc.body.split("\n")):
m = HEADING.match(raw)
if m:
out.append((doc.body_start + i, len(m.group(1)), (m.group(2) or "").strip()))
return out
def check_spec11_index(b: Bundle) -> list[Finding]:
"""SPEC 8 via 11.3: no frontmatter except `okf_version` at the bundle root; entries grouped under headings."""
out: list[Finding] = []
for d in b.named("index.md"):
if d.has_frontmatter:
at_root = d.rel == "index.md"
if d.frontmatter is None:
out.append(finding("spec11-3-index", d.rel, 1, d.frontmatter_error + " (SPEC 8)"))
elif not at_root or set(d.frontmatter) - {"okf_version"}:
out.append(finding("spec11-3-index", d.rel, 1,
"an index file carries no frontmatter, except okf_version in the bundle-root index (SPEC 8)"))
body_lines = d.body.split("\n")
headings = body_headings(d)
first_heading = headings[0][0] if headings else None
before = [d.body_start + i for i, raw in enumerate(body_lines)
if raw.strip() and (first_heading is None or d.body_start + i < first_heading)]
if not headings:
out.append(finding("spec11-3-index", d.rel, d.body_start, "no section heading; entries are grouped under headings (SPEC 8)"))
elif before:
out.append(finding("spec11-3-index", d.rel, before[0], "content before the first section heading (SPEC 8)"))
return out
DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
def check_spec11_log(b: Bundle) -> list[Finding]:
"""SPEC 9 via 11.3: date headings in YYYY-MM-DD form, entries under a date, newest first."""
out: list[Finding] = []
for d in b.named("log.md"):
dates: list[tuple[int, date]] = []
first_date_line: int | None = None
for line, level, text in body_headings(d):
if level == 1:
continue
if not DATE.match(text):
out.append(finding("spec11-3-log", d.rel, line, f"date heading {text!r} is not ISO 8601 YYYY-MM-DD (SPEC 9)"))
continue
try:
dates.append((line, date.fromisoformat(text)))
except ValueError:
out.append(finding("spec11-3-log", d.rel, line, f"date heading {text!r} is not a calendar date (SPEC 9)"))
continue
if first_date_line is None:
first_date_line = line
for (_, newer), (line, older) in zip(dates, dates[1:]):
if older > newer:
out.append(finding("spec11-3-log", d.rel, line, "entries are not newest first (SPEC 9)"))
for i, raw in enumerate(d.body.split("\n")):
line = d.body_start + i
if LIST_ITEM.match(raw) and (first_date_line is None or line < first_date_line):
out.append(finding("spec11-3-log", d.rel, line, "an entry that is not under a date heading (SPEC 9)"))
break
return out
def check_links(b: Bundle) -> list[Finding]:
"""broken-link, orphan-page and index-coverage, as defined in the manifest's provenance.link_rules."""
out: list[Finding] = []
incoming: dict[str, set[str]] = {d.rel: set() for d in b.concepts}
for d in b.concepts:
for line, target in body_links(d):
if not is_bundle_link(target):
continue
rel = resolve(d.rel, target)
if not b.is_file(rel):
out.append(finding("broken-link", d.rel, line, f"link target {target!r} is not a file in the bundle"))
elif rel in incoming and rel != d.rel:
incoming[rel].add(d.rel)
for idx in b.named("index.md"):
listed: set[str] = set()
for line, target in body_links(idx):
if not is_bundle_link(target):
continue
rel = resolve(idx.rel, target)
if rel is not None and rel != "" and (b.root / rel).is_dir():
continue
if not b.is_file(rel):
out.append(finding("index-coverage", idx.rel, line, f"index entry {target!r} is not a file"))
continue
assert rel is not None
listed.add(rel)
if rel in incoming:
incoming[rel].add(idx.rel)
directory = PurePosixPath(idx.rel).parent
for d in b.concepts:
if PurePosixPath(d.rel).parent == directory and d.rel not in listed:
out.append(finding("index-coverage", idx.rel, None, f"{d.rel} is in the directory but not in its index (SPEC 8)"))
for rel, sources in sorted(incoming.items()):
if not sources:
out.append(finding("orphan-page", rel, None, "no other concept and no index.md links to this page (log.md does not count)"))
return out
def placed_links(doc: Document) -> list[tuple[str, int, str]]:
"""(place, file line, target) for every inline markdown link in the body; place is prose, code, heading or footnote.
Code is what mask_code blanks (inline spans, and blocks fenced with backticks) and every line of a fenced block,
backticks or tildes, fences included, also when the fence is indented in a list item or stands behind > in a quote
block (K2 point 14). body_links() is the prose and heading/footnote part.
"""
masked = mask_code(doc.body) # same length as the body, so index by offset
out: list[tuple[str, int, str]] = []
offset = 0
fence = "" # the open fence's marker run, or "" outside a fenced block
for i, raw in enumerate(doc.body.split("\n")):
f = fence_marker(raw)
in_fence = bool(fence or f)
if fence and closes_fence(f, raw, fence):
fence = ""
elif not fence and f:
fence = f.group(1)
place = "heading" if HEADING.match(raw) else "footnote" if FOOTNOTE_DEF.match(raw.strip()) else "prose"
for m in link_matches(raw, masked[offset:offset + len(raw)]):
at = offset + m.start()
in_code = in_fence or masked[at] != doc.body[at] # the link opens inside code; code inside the anchor is fine
out.append(("code" if in_code else place, doc.body_start + i, m.group(2)))
offset += len(raw) + 1
return out
def concept_inbound(b: Bundle) -> tuple[dict[str, set[str]], list[Finding]]:
"""Criterion 6 (b) and (c): inbound concept links per concept page counted from prose only, and misplaced links."""
incoming: dict[str, set[str]] = {d.rel: set() for d in b.concepts}
misplaced: list[Finding] = []
for d in b.concepts:
for place, line, target in placed_links(d):
if not is_bundle_link(target):
continue
if place != "prose":
misplaced.append(finding("link-misplaced", d.rel, line,
f"link to {target!r} sits in {place}; a link belongs in a body sentence (criterion 6 b)"))
continue
rel = resolve(d.rel, target)
if rel in incoming and rel != d.rel:
incoming[rel].add(d.rel)
return incoming, misplaced
def check_link_structure(b: Bundle) -> list[Finding]:
"""link-misplaced (b) of criterion 6; (a) is broken-link in check_links, and the main-page rule is check_main_pages."""
return concept_inbound(b)[1]
def link_rule_report(b: Bundle) -> dict[str, Any]:
"""Criterion 6 as numbers: N of N concept pages with an inbound link, the breaches of (a) and (b), and counts."""
incoming, misplaced = concept_inbound(b)
concepts = set(incoming)
passed = sum(1 for s in incoming.values() if s)
not_to_concept = 0
out_degree: dict[str, set[str]] = {rel: set() for rel in concepts}
provenance: dict[str, int] = {}
for d in b.concepts:
for place, _, target in placed_links(d):
if place != "prose" or not is_bundle_link(target):
continue
rel = resolve(d.rel, target)
if rel in concepts and rel != d.rel:
out_degree[d.rel].add(rel)
elif rel not in concepts and b.is_file(rel):
not_to_concept += 1
srcs = d.frontmatter.get("sources") if d.frontmatter else None
for s in srcs if isinstance(srcs, list) else []:
value = s.get("x-provenance") if isinstance(s, dict) else None
key = value if isinstance(value, str) else "missing"
provenance[key] = provenance.get(key, 0) + 1
return {
"pass": passed,
"of": len(concepts),
"line": f"link rule: {passed} of {len(concepts)} concept pages have an inbound link",
"misplaced": len(misplaced),
"broken": sum(f.check == "broken-link" for f in check_links(b)),
"not_to_concept": not_to_concept,
"max_out_degree": max((len(v) for v in out_degree.values()), default=0),
"provenance": dict(sorted(provenance.items())),
}
# --- the main-page rule (contract K2, read against K1) -------------------------------------------------------------
K1_FILE = "source-manifest.json"
K1_SCHEMA = "okf-wiki/source-manifest/1"
MAIN_FOR = "x-main-for"
class K1Error(Exception):
"""The K1 file at the sources root is not valid, so the main-page rule cannot run (K2 point 18)."""
def read_k1(sources: Path) -> dict[str, bool] | None:
"""`main_topic` per `path` from <sources>/source-manifest.json (contract K1), read by this module itself and not
by the K1 module, which wave 2 changes beside it. None when there is no such file. K1Error when it is not valid:
not JSON, `schema` not okf-wiki/source-manifest/1, `sources` not a list, or an entry without `path` (a string) or
`main_topic` (a boolean). Unknown keys are ignored (K1 point 10)."""
path = sources / K1_FILE
if not path.exists():
return None
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, ValueError) as exc:
raise K1Error(f"{path} is not readable JSON: {exc}") from exc
if not isinstance(data, dict) or data.get("schema") != K1_SCHEMA:
raise K1Error(f"{path}: schema is not {K1_SCHEMA}")
entries = data.get("sources")
if not isinstance(entries, list):
raise K1Error(f"{path}: sources is not a list")
out: dict[str, bool] = {}
for i, e in enumerate(entries):
if not (isinstance(e, dict) and isinstance(e.get("path"), str) and isinstance(e.get("main_topic"), bool)):
raise K1Error(f"{path}: sources[{i}] has no path (a string) or main_topic (a boolean)")
out[e["path"]] = e["main_topic"]
return out
def main_marks(b: Bundle, k1: dict[str, bool]) -> tuple[dict[str, set[str]], list[Finding]]:
"""The valid x-main-for values per page, and one main-page-invalid finding per invalid value (K2 points 1, 2, 10-13).
A valid value is a string that stands, exactly, among the page's own sources[].resource and is "/" + a path with
main_topic true in K1. A value that is not a list, or an empty list, is one finding and gives no role; in a list,
each item is judged alone, and a value given twice counts once and gives one finding. A page is a main page when at
least one value is valid. A page whose frontmatter does not parse has no role (point 16).
"""
topics = {"/" + p for p, main in k1.items() if main}
valid: dict[str, set[str]] = {}
out: list[Finding] = []
for d in b.concepts:
fm = d.frontmatter
if fm is None or MAIN_FOR not in fm:
continue
value = fm[MAIN_FOR]
if not isinstance(value, list):
out.append(finding("main-page-invalid", d.rel, None, f"{MAIN_FOR} {value!r} is not a list (K2 point 1)"))
continue
if not value:
out.append(finding("main-page-invalid", d.rel, None,
f"{MAIN_FOR} is an empty list; a page that is no main page leaves the key out (K2 point 11)"))
continue
srcs = fm.get("sources")
own = {s["resource"] for s in (srcs if isinstance(srcs, list) else [srcs])
if isinstance(s, dict) and isinstance(s.get("resource"), str)}
seen: set[str] = set()
for v in value:
if not isinstance(v, str):
why = "is not a string (K2 point 10)"
elif v in seen:
why = "is listed twice (K2 point 12)"
elif v not in own:
why = "is not among the page's own sources[].resource (K2 points 1 and 13)"
elif v not in topics:
why = "is not / + a path with main_topic true in K1 (K2 points 1 and 13)"
else:
why = ""
valid.setdefault(d.rel, set()).add(v)
if isinstance(v, str):
seen.add(v)
if why:
out.append(finding("main-page-invalid", d.rel, None, f"{MAIN_FOR} value {v!r} {why}"))
return valid, out
def main_holders(valid: dict[str, set[str]], k1: dict[str, bool]) -> dict[str, list[str]]:
"""For every K1 path with main_topic true, the pages that list it validly in x-main-for (valid: main_marks)."""
return {p: sorted(rel for rel, values in valid.items() if "/" + p in values) for p, main in sorted(k1.items()) if main}
def main_page_links(b: Bundle, mains: set[str]) -> dict[str, set[str]]:
"""For every concept page, the main pages whose body links to it (K2 points 3, 14 and 15).
Only inline links with a relative target count, in prose: paragraphs, list items, table cells and quote blocks
under any heading, outside code (inline and fenced), headings and footnote definitions. A link with #anchor counts
for its file; a link starting with / does not; a link from the page itself does not. Reference links and autolinks
are no inline links, so they do not count.
"""
linked: dict[str, set[str]] = {d.rel: set() for d in b.concepts}
for d in b.concepts:
if d.rel not in mains:
continue
for place, _, target in placed_links(d):
if place != "prose" or not is_bundle_link(target) or target.startswith("/"):
continue
rel = resolve(d.rel, target)
if rel in linked and rel != d.rel:
linked[rel].add(d.rel)
return linked
def check_main_pages(b: Bundle, k1: dict[str, bool]) -> list[Finding]:
"""K2 point 5: main-page-missing and main-page-duplicate per main-topic source (the finding's file is its K1 path),
main-page-invalid per invalid value, and no-main-page-link per page that is not a main page and has no link from one."""
valid, out = main_marks(b, k1)
for path, pages in main_holders(valid, k1).items():
if not pages:
out.append(finding("main-page-missing", path, None,
f"main-topic source /{path} has no main page: no concept page lists it in {MAIN_FOR} (K2 point 2)"))
elif len(pages) > 1:
out.append(finding("main-page-duplicate", path, None,
f"main-topic source /{path} has {len(pages)} main pages: {', '.join(pages)} (K2 point 2)"))
for rel, mains in sorted(main_page_links(b, set(valid)).items()):
if rel not in valid and not mains:
out.append(finding("no-main-page-link", rel, None,
"no main page links to this page from its body, in a sentence or a list of related concepts; "
"index.md, log.md and the page itself do not count (K2 point 3)"))
return out
def main_page_report(b: Bundle, k1: dict[str, bool]) -> dict[str, Any]:
"""Criterion 6 as numbers (K2 point 7): the main-topic sources with exactly one main page, the pages that are not
main pages with a link from one, and the counts of link_rule_report beside them."""
valid, _ = main_marks(b, k1)
holders = main_holders(valid, k1)
sources = sum(1 for pages in holders.values() if len(pages) == 1)
linked = main_page_links(b, set(valid))
others = [rel for rel in linked if rel not in valid]
passed = sum(1 for rel in others if linked[rel])
counts = link_rule_report(b)
return {
"line": f"main pages: {sources} of {len(holders)} sources · linked from a main page: {passed} of {len(others)} pages",
"main_pages": {"pass": sources, "of": len(holders)},
"linked": {"pass": passed, "of": len(others)},
**{k: counts[k] for k in ("misplaced", "broken", "not_to_concept", "max_out_degree", "provenance")},
}
def as_events(value: Any) -> list[Any]:
"""`verified` as a list of events; a bare mapping is a one-element list (SPEC 5.2)."""
if isinstance(value, dict):
return [value]
if isinstance(value, list):
return value
return [value]
TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:\d{2})$")
def timestamp_ok(value: Any) -> bool:
"""ISO 8601 datetime with an explicit UTC offset (SPEC 5), kept as the text the author wrote (format choice c)."""
if not isinstance(value, str) or not TIMESTAMP.match(value):
return False
try:
datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return False
return True
def timestamp_values(fm: dict[str, Any]) -> list[tuple[str, Any]]:
"""Every OKF timestamp-valued key present: generated.at, verified[].at, stale_after, sources[].last_modified,
and the from/to of usage_window, shared or per source (SPEC 5.1, 5.2, 5.5)."""
out: list[tuple[str, Any]] = []
def window(prefix: str, w: Any) -> None:
if isinstance(w, dict):
out.extend((f"{prefix}.{k}", w[k]) for k in ("from", "to") if k in w)
gen = fm.get("generated")
if isinstance(gen, dict) and "at" in gen:
out.append(("generated.at", gen["at"]))
if "verified" in fm:
out.extend(("verified[].at", e["at"]) for e in as_events(fm["verified"]) if isinstance(e, dict) and "at" in e)
if "stale_after" in fm:
out.append(("stale_after", fm["stale_after"]))
window("usage_window", fm.get("usage_window"))
srcs = fm.get("sources")
if isinstance(srcs, list):
for s in srcs:
if isinstance(s, dict):
if "last_modified" in s:
out.append(("sources[].last_modified", s["last_modified"]))
window("sources[].usage_window", s.get("usage_window"))
return out
def check_format(b: Bundle) -> list[Finding]:
"""Format choices (a) resource with a leading /, (b) relative body links, (c) timestamps, (e) verified by humans."""
out: list[Finding] = []
for d in b.concepts:
for line, target in body_links(d):
if is_bundle_link(target) and target.startswith("/"):
out.append(finding("body-link-bundle-absolute", d.rel, line,
f"body link {target!r} starts with /; body links are relative (format choice b)"))
fm = d.frontmatter
if fm is None:
continue
srcs = fm.get("sources")
for s in srcs if isinstance(srcs, list) else []:
res = s.get("resource") if isinstance(s, dict) else None
if isinstance(res, str) and res and not SCHEME.match(res) and not res.startswith("/"):
out.append(finding("resource-not-bundle-absolute", d.rel, None,
f"sources[].resource {res!r} has no leading / (format choice a)"))
if "verified" in fm:
for e in as_events(fm["verified"]):
by = e.get("by") if isinstance(e, dict) else None
if not (isinstance(by, str) and by.startswith("human:") and len(by) > len("human:")):
out.append(finding("verified-not-human", d.rel, None,
f"verified by {by!r}: only a human:<id> may write verified (format choice e)"))
for key, value in timestamp_values(fm):
if not timestamp_ok(value):
out.append(finding("timestamp-form", d.rel, None,
f"{key} {value!r} is not an ISO 8601 datetime with an explicit UTC offset (SPEC 5)"))
return out
def check_citations(b: Bundle) -> list[Finding]:
"""Criterion 1 and format choice f: sentence -> footnote -> sources[].id -> verbatim quote at its locator."""
out: list[Finding] = []
source_lines: dict[str, list[str] | None] = {}
for d in b.concepts:
for block in prose_blocks(d):
if LINK_ONLY.match(mask_code(block.text)):
continue # a pure link list is not a claim
for s in sentences(block.text):
if not FOOTNOTE_REF.search(s):
out.append(finding("uncited-sentence", d.rel, block.line, f"sentence without a footnote: {s[:80]!r}"))
fm = d.frontmatter
if fm is None:
continue # without parseable frontmatter there are no sources to join to (spec11-1 reports it)
srcs = fm.get("sources", [])
entries = srcs if isinstance(srcs, list) else [srcs]
ids: list[str] = []
for s in entries:
sid = s.get("id") if isinstance(s, dict) else None
if not isinstance(sid, str) or not sid.strip():
out.append(finding("source-missing-id", d.rel, None, "a sources entry has no id (format choice f)"))
else:
ids.append(sid)
for sid in sorted({i for i in ids if ids.count(i) > 1}):
out.append(finding("source-duplicate-id", d.rel, None, f"sources[].id {sid!r} is used more than once (format choice f)"))
known = set(ids)
labels: dict[str, int] = {}
for i, raw in masked_lines(d.body):
m = FOOTNOTE_DEF.match(raw.strip())
found = [m.group(1)] if m else FOOTNOTE_REF.findall(raw)
for label in found:
labels.setdefault(label, d.body_start + i)
for label, line in sorted(labels.items(), key=lambda kv: kv[1]):
if label not in known:
out.append(finding("footnote-unknown-id", d.rel, line, f"footnote [^{label}] names no sources[].id"))
for s in entries:
if isinstance(s, dict):
out.extend(check_quote(b, d, s, source_lines))
return out
PLACE_KEYS = frozenset({"page", "paragraph", "sheet", "cell", "slide", "notes"})
def check_quote(b: Bundle, d: Document, s: dict[str, Any], cache: dict[str, list[str] | None]) -> list[Finding]:
sid = s.get("id", "?")
quote, locator, res = s.get("x-quote"), s.get("x-locator"), s.get("resource")
if not isinstance(quote, str) or not quote:
return [finding("quote-missing", d.rel, None, f"source {sid!r} has no x-quote")]
if not (isinstance(locator, dict) and "line" in locator and set(locator) - {"line"} <= PLACE_KEYS
and type(locator["line"]) is int and locator["line"] > 0):
return [finding("locator-unsupported", d.rel, None, f"source {sid!r}: x-locator {locator!r} is not {{ line: N }}"
" with at most a page, paragraph, sheet and cell, or slide and notes")]
if not isinstance(res, str) or not res or SCHEME.match(res):
return [finding("source-unreadable", d.rel, None, f"source {sid!r}: resource {res!r} is not a raw-source path")]
rel = resolve(d.rel, res)
if not rel:
return [finding("source-unreadable", d.rel, None, f"source {sid!r}: resource {res!r} leaves the raw-source root")]
if rel not in cache:
try:
cache[rel] = (b.sources / rel).read_text(encoding="utf-8").splitlines()
except (OSError, UnicodeDecodeError):
cache[rel] = None
lines = cache[rel]
if lines is None:
return [finding("source-unreadable", d.rel, None, f"source {sid!r}: cannot read {res!r} as UTF-8 text")]
n = locator["line"]
if n <= len(lines) and quote in lines[n - 1]:
place = {k: v for k, v in locator.items() if k != "line"}
places = b.places(rel)
frozen = places[n - 1] if places is not None and n <= len(places) else None
if not place_fits(place, frozen):
return [finding("quote-wrong-place", d.rel, None, f"source {sid!r}: line {n} of {res!r} is at "
f"{frozen or 'no place'} in the original, not {place}")]
return []
if any(quote in line for line in lines):
return [finding("quote-wrong-locator", d.rel, None, f"source {sid!r}: the quote is in {res!r}, but not on line {n}")]
return [finding("quote-not-verbatim", d.rel, None, f"source {sid!r}: the quote is not verbatim anywhere in {res!r}")]
def audit(bundle: Path, sources: Path) -> list[Finding]:
"""Every finding on the bundle; the main-page rule only when SOURCES holds a K1 file (K1Error when it is invalid)."""
k1 = read_k1(sources)
return check_bundle(load_bundle(bundle, sources), k1)
def check_bundle(b: Bundle, k1: dict[str, bool] | None) -> list[Finding]:
found = (check_spec11_concepts(b) + check_spec11_index(b) + check_spec11_log(b)
+ check_links(b) + check_link_structure(b) + check_citations(b) + check_format(b))
if k1 is not None:
found += check_main_pages(b, k1)
return sorted(found, key=lambda f: (GROUPS.index(f.group), f.file, f.line or 0, f.check, f.message))
def main(argv: list[str] | None = None) -> int:
args = sys.argv[1:] if argv is None else argv
if len(args) != 2:
print("usage: python3 -m okfwiki.audit BUNDLE SOURCES", file=sys.stderr)
return 2
bundle, sources = Path(args[0]), Path(args[1])
for p, what in ((bundle, "bundle"), (sources, "raw-source root")):
if not p.is_dir():
print(f"audit cannot run: {what} {p} is not a directory", file=sys.stderr)
return 2
try:
k1 = read_k1(sources)
except K1Error as exc:
print(f"audit cannot run: {exc}; the main-page rule needs a valid K1 file (K2 points 6 and 18)", file=sys.stderr)
return 2
if k1 is None:
print(f"audit cannot run: no {K1_FILE} at the raw-source root {sources}; the main-page rule needs K1 (K2 point 6)",
file=sys.stderr)
return 2
b = load_bundle(bundle, sources)
if not any(k1.values()) and not b.concepts:
print(f"audit cannot run: {K1_FILE} lists no main-topic source and the bundle has no concept page, so the "
"main-page rule has no denominator", file=sys.stderr)
return 2
found = check_bundle(b, k1)
summary = {g: sum(f.group == g for f in found) for g in GROUPS}
print(json.dumps({"findings": [asdict(f) for f in found], "summary": summary, "link_rule": main_page_report(b, k1)},
ensure_ascii=False, indent=1))
return 1 if found else 0
if __name__ == "__main__":
sys.exit(main())

467
okfwiki/claims.py Normal file
View file

@ -0,0 +1,467 @@
"""The claim list behind PLAN.md criterion 1 ("sitat ved plassering") and 2 ("slutning støttet"), and the reading
layout for pages (K6 in the wave-2 contracts). It calls no model.
The denominator comes from the pages, not from the footnotes: every sentence in the body of a concept page is a claim.
Headings and pure link lists are not claims; they are listed as exceptions so the count can be checked by hand. Every
claim has a stable id "<page>#<key>", the key being klink's sentence key (okfwiki.klink.page_sentences) for the same
sentence at the same place, so the id survives a new assembly of the page; a claim klink does not see is listed as
unkeyed and counts as failing, never as a claim without an id. Every claim carries excerpt_sha256 (the excerpts it shows
the reader, in order) and its provenance mark: the strongest x-provenance among the entries it cites, missing > merged >
inferred > documented, "missing" when an entry has no valid mark or the claim cites none (so it is read, never sampled).
list BUNDLE SOURCES --out CLAIMS [--pairs PAIRS]
Criterion 1: a claim counts when it has at least one footnote and every footnote names a sources[].id whose
x-quote stands verbatim at its x-locator (the auditor's own quote check). With --pairs, the reader set
({"pairs": [{id, claim, excerpts: [{file, line, text}]}]}); a claim with no quote to show is left out.
calls --claims CLAIMS --pairs PAIRS --out CALLS [--ids IDS] [--size N]
The call layout (okf-wiki/reader-calls/1) from the pages: the claims of one page stay together in calls of
at most ten (at most N with --size, 1 to 10), in page order; only claims in PAIRS, and with --ids only
those listed.
sample --claims CLAIMS --judged JUDGED --part-root ROOT --seed N --min 300 --out IDS
The claims to read after the first part: a claim is judged when its id and excerpt_sha256 stand in JUDGED
(the claim list of an earlier reading). Read are every claim not judged that cites a source of the part
(ROOT's source-manifest.json) or is marked inferred, merged or missing, and a seeded draw of at least --min
of the others. Written with the seed, the sha256 of the inputs and of the ids, and the error bound.
sample --claims CLAIMS --of-all --seed N --min 300 --out IDS
The reading after a build changes (the operator 13.11.2026): a seeded draw of --min among every claim,
whatever its mark and whether it was judged before, and nothing added to it.
score --claims CLAIMS --verdicts VERDICTS [--ids IDS]
Criterion 2 over the same claims (or the ids): a claim counts when its verdict is "supported"; a missing or
invalid verdict counts as not supported (fail-closed). Not supported is counted per provenance mark, and for
a sample the upper one-sided 95 % Clopper-Pearson bound is printed (0 errors in 299 is under 1 %).
Exit 0 = N of N (or written), 1 = at least one claim does not count (or nothing to write), 2 = could not run.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import random
import sys
from dataclasses import asdict, dataclass, field
from pathlib import Path, PurePosixPath
from typing import Any
from okfwiki.audit import (
FOOTNOTE_REF,
LINK_ONLY,
Bundle,
Document,
body_headings,
check_quote,
load_bundle,
mask_code,
prose_blocks,
sentence_spans,
)
from okfwiki.klink import key_text, page_sentences
from okfwiki.reader import SUPPORTED, load_pairs
PROVENANCE = ("documented", "inferred", "merged")
MISSING = "missing"
STRENGTH = {"documented": 0, "inferred": 1, "merged": 2, MISSING: 3}
MUST_READ = ("inferred", "merged", MISSING)
CALL_SIZE = 10
SAMPLE_MIN = 300
@dataclass(frozen=True)
class Excerpt:
file: str
line: int
text: str
@dataclass
class Claim:
id: str
page: str
line: int
claim: str
footnotes: list[str]
excerpts: list[Excerpt]
quote_ok: bool
excerpt_sha256: str = ""
provenance: str = MISSING
resources: list[str] = field(default_factory=list)
@dataclass(frozen=True)
class Exempt:
page: str
line: int
kind: str # "heading" or "link-list"
text: str
@dataclass(frozen=True)
class Unkeyed:
page: str
line: int
text: str
@dataclass
class ClaimList:
claims: list[Claim] = field(default_factory=list)
exceptions: list[Exempt] = field(default_factory=list)
unkeyed: list[Unkeyed] = field(default_factory=list)
def excerpt_digest(excerpts: list[Excerpt]) -> str:
"""sha256 of the excerpts a claim shows the reader, in order (file, line, text)."""
rows = [[e.file, e.line, e.text] for e in excerpts]
return hashlib.sha256(json.dumps(rows, ensure_ascii=False, separators=(",", ":")).encode("utf-8")).hexdigest()
def provenance_of(doc: Document, labels: list[str]) -> str:
"""The strongest mark among the cited entries; missing when an entry has no valid mark or nothing is cited."""
by_id = _sources_by_id(doc)
marks = [by_id[x].get("x-provenance") if x in by_id else None for x in labels] or [None]
return max((m if isinstance(m, str) and m in PROVENANCE else MISSING for m in marks), key=STRENGTH.__getitem__)
def _digest(text: str) -> str:
return hashlib.sha256(key_text(text).encode("utf-8")).hexdigest()[:16]
def _sources_by_id(doc: Document) -> dict[str, dict[str, Any]]:
fm = doc.frontmatter or {}
srcs = fm.get("sources", [])
entries = srcs if isinstance(srcs, list) else [srcs]
out: dict[str, dict[str, Any]] = {}
for s in entries:
if isinstance(s, dict) and isinstance(s.get("id"), str):
out.setdefault(s["id"], s)
return out
def _judge(b: Bundle, doc: Document, labels: list[str], cache: dict[str, list[str] | None]) -> tuple[list[Excerpt], bool]:
"""The excerpts a claim cites, and whether every one of them stands verbatim at its locator."""
by_id = _sources_by_id(doc)
excerpts: list[Excerpt] = []
ok = bool(labels)
for label in labels:
s = by_id.get(label)
if s is None:
ok = False
continue
if check_quote(b, doc, s, cache):
ok = False
quote, locator, res = s.get("x-quote"), s.get("x-locator"), s.get("resource")
if isinstance(quote, str) and quote and isinstance(res, str) and isinstance(locator, dict) \
and type(locator.get("line")) is int:
e = Excerpt(PurePosixPath(res).name, locator["line"], quote)
if e not in excerpts:
excerpts.append(e)
return excerpts, ok
def extract(bundle: Path, sources: Path) -> ClaimList:
b = load_bundle(bundle, sources)
out = ClaimList()
cache: dict[str, list[str] | None] = {}
for doc in b.concepts:
keys = [x.key for x in page_sentences((bundle / doc.rel).read_text(encoding="utf-8"))]
at = 0
for line, _level, text in body_headings(doc):
out.exceptions.append(Exempt(doc.rel, line, "heading", text))
for block in prose_blocks(doc):
if LINK_ONLY.match(mask_code(block.text)):
out.exceptions.append(Exempt(doc.rel, block.line, "link-list", block.text))
continue
for a, z in sentence_spans(block.text):
sentence = block.text[a:z]
labels: list[str] = []
for label in FOOTNOTE_REF.findall(mask_code(sentence)):
if label not in labels:
labels.append(label)
text = FOOTNOTE_REF.sub("", sentence).strip()
digest = _digest(sentence)
found = next((i for i in range(at, len(keys)) if keys[i].rsplit("-", 1)[0] == digest), None)
if found is None:
out.unkeyed.append(Unkeyed(doc.rel, block.line, text))
continue
at = found + 1
excerpts, ok = _judge(b, doc, labels, cache)
resources = [str(r) for r in (_sources_by_id(doc).get(x, {}).get("resource") for x in labels)
if isinstance(r, str)]
out.claims.append(Claim(f"{doc.rel}#{keys[found]}", doc.rel, block.line, text, labels, excerpts, ok,
excerpt_digest(excerpts), provenance_of(doc, labels),
list(dict.fromkeys(resources))))
return out
Tally = tuple[int, int]
def _tally(items: list[tuple[list[str], bool]]) -> dict[str, Tally]:
counts: dict[str, list[int]] = {}
for keys, ok in items:
for k in keys:
c = counts.setdefault(k, [0, 0])
c[0] += int(ok)
c[1] += 1
return {k: (v[0], v[1]) for k, v in sorted(counts.items())}
def _report(found: ClaimList, ok: dict[str, bool], cs: list[Claim] | None = None, extra: int = 0) -> dict[str, Any]:
"""Totals per page, per cited file and per provenance mark; `extra` claims that cannot count (unkeyed claims,
ids that are not claims) are added to the denominator as failing."""
cs = found.claims if cs is None else cs
return {
"total": (sum(ok[c.id] for c in cs), len(cs) + extra),
"per_page": _tally([([c.page], ok[c.id]) for c in cs]),
"per_file": _tally([(sorted({e.file for e in c.excerpts}), ok[c.id]) for c in cs]),
"not_supported_per_mark": {k: (b - a, b)
for k, (a, b) in _tally([([c.provenance], ok[c.id]) for c in cs]).items()},
"not_supported": sorted(c.id for c in cs if not ok[c.id]),
}
def criterion_1(found: ClaimList) -> dict[str, Any]:
return _report(found, {c.id: c.quote_ok for c in found.claims}, extra=len(found.unkeyed))
def criterion_2(found: ClaimList, verdicts: dict[str, Any], ids: list[str] | None = None) -> dict[str, Any]:
given: dict[str, list[str]] = {}
for v in verdicts.get("verdicts", []) if isinstance(verdicts, dict) else []:
if isinstance(v, dict) and isinstance(v.get("id"), str):
given.setdefault(v["id"], []).append(str(v.get("verdict")))
ok = {c.id: given.get(c.id) == [SUPPORTED] for c in found.claims}
if ids is None:
return _report(found, ok, extra=len(found.unkeyed))
wanted = set(ids)
chosen = [c for c in found.claims if c.id in wanted]
return _report(found, ok, chosen, extra=len(wanted) - len(chosen))
def error_bound(k: int, n: int) -> float:
"""The upper one-sided 95 % Clopper-Pearson bound on the error rate after k errors in n: the p at which k or fewer
errors in n have probability 0.05. For k = 0 it is 1 - 0.05 ** (1 / n)."""
if n <= 0 or k >= n:
return 1.0
def at_most_k(p: float) -> float:
return sum(math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1)
+ i * math.log(p) + (n - i) * math.log1p(-p)) for i in range(k + 1))
lo, hi = 0.0, 1.0
for _ in range(200):
mid = (lo + hi) / 2
if mid <= 0.0 or at_most_k(mid) > 0.05:
lo = mid
else:
hi = mid
return hi
def write_pairs(found: ClaimList, out: Path) -> None:
pairs = [{"id": c.id, "claim": c.claim, "excerpts": [asdict(e) for e in c.excerpts]}
for c in found.claims if c.excerpts]
out.write_text(json.dumps({"schema": "okf-wiki/claim-pairs/1", "pairs": pairs}, indent=1, ensure_ascii=False) + "\n",
encoding="utf-8")
def load_claims(path: Path) -> ClaimList:
data = json.loads(path.read_text(encoding="utf-8"))
found = ClaimList()
for c in data["claims"]:
excerpts = [Excerpt(**e) for e in c["excerpts"]]
found.claims.append(Claim(c["id"], c["page"], c["line"], c["claim"], list(c["footnotes"]), excerpts,
bool(c["quote_ok"]), str(c.get("excerpt_sha256") or excerpt_digest(excerpts)),
str(c.get("provenance", MISSING)), list(c.get("resources", []))))
for e in data.get("exceptions", []):
found.exceptions.append(Exempt(**e))
for u in data.get("unkeyed", []):
found.unkeyed.append(Unkeyed(**u))
return found
_load = load_claims
def _sha(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _dump(doc: Any) -> str:
return json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n"
def _ids(path: Path) -> list[str]:
data = json.loads(path.read_text(encoding="utf-8"))
ids = data.get("ids") if isinstance(data, dict) else None
if not isinstance(ids, list) or not all(isinstance(i, str) for i in ids):
raise ValueError(f"{path} has no list of ids")
return ids
def _print(name: str, report: dict[str, Any]) -> None:
x, n = report["total"]
print(f"{name}: {x} of {n}")
for key in ("per_page", "per_file"):
print(f" {key.replace('_', ' ')}:")
for k, (a, b) in report[key].items():
print(f" {k}: {a} of {b}")
def cmd_list(args: argparse.Namespace) -> int:
bundle, sources = Path(args.bundle), Path(args.sources)
for p, what in ((bundle, "bundle"), (sources, "raw-source root")):
if not p.is_dir():
print(f"claims cannot run: {what} {p} is not a directory", file=sys.stderr)
return 2
found = extract(bundle, sources)
doc = {"schema": "okf-wiki/claims/1",
"claims": [asdict(c) for c in found.claims],
"exceptions": [asdict(e) for e in found.exceptions],
"unkeyed": [asdict(u) for u in found.unkeyed]}
Path(args.out).write_text(json.dumps(doc, indent=1, ensure_ascii=False) + "\n", encoding="utf-8")
if args.pairs:
write_pairs(found, Path(args.pairs))
report = criterion_1(found)
_print("criterion 1, quote at locator", report)
kinds = {k: sum(e.kind == k for e in found.exceptions) for k in ("heading", "link-list")}
print(f"exceptions (not claims): {kinds['heading']} headings, {kinds['link-list']} link lists")
print(f"claims without a sentence key (counted as failing): {len(found.unkeyed)}")
x, n = report["total"]
return 0 if n and x == n else 1
def cmd_calls(args: argparse.Namespace) -> int:
size: int = args.size
if not 1 <= size <= CALL_SIZE:
raise ValueError(f"--size {size} is not a whole number from 1 to {CALL_SIZE}; nothing written")
found = load_claims(Path(args.claims))
pairs_path = Path(args.pairs)
with_pair = {p.id for p in load_pairs(pairs_path)}
wanted = set(_ids(Path(args.ids))) if args.ids else None
chosen = [c for c in found.claims if (wanted is None or c.id in wanted) and c.id in with_pair]
left_out = (len(wanted) if wanted is not None else len(found.claims)) - len(chosen)
pages: dict[str, list[str]] = {}
for c in chosen:
pages.setdefault(c.page, []).append(c.id)
groups = [ids[i : i + size] for ids in pages.values() for i in range(0, len(ids), size)]
if not groups:
print("calls: no claim to lay out", file=sys.stderr)
return 1
layout = {"call_size_max": size, "calls": [{"call": f"k{n:03d}", "ids": g} for n, g in enumerate(groups, 1)],
"schema": "okf-wiki/reader-calls/1", "set_sha256": _sha(pairs_path)}
Path(args.out).write_text(_dump(layout), encoding="utf-8")
print(f"calls: {len(chosen)} claims from {len(pages)} pages in {len(groups)} calls; "
f"{left_out} left out (no excerpt to show, or not a claim)")
return 0
def cmd_sample(args: argparse.Namespace) -> int:
if args.min < SAMPLE_MIN:
raise ValueError(f"--min {args.min} is under {SAMPLE_MIN}, the least sample K6 point 4 allows")
claims_path = Path(args.claims)
found = load_claims(claims_path)
judged_sha: str | None = None
manifest_sha: str | None = None
if args.of_all:
if args.judged or args.part_root:
raise ValueError("--of-all draws among every claim and takes no --judged or --part-root")
open_, must = list(found.claims), []
else:
if not args.judged or not args.part_root:
raise ValueError("sample needs --judged and --part-root, or --of-all to draw among every claim")
judged_path, manifest = Path(args.judged), Path(args.part_root) / "source-manifest.json"
part = {str(s["path"]) for s in json.loads(manifest.read_text(encoding="utf-8"))["sources"]}
done = {(c.id, c.excerpt_sha256) for c in load_claims(judged_path).claims}
open_ = [c for c in found.claims if (c.id, c.excerpt_sha256) not in done]
must = [c.id for c in open_ if c.provenance in MUST_READ or any(r.lstrip("/") in part for r in c.resources)]
judged_sha, manifest_sha = _sha(judged_path), _sha(manifest)
others = [c.id for c in open_ if c.id not in set(must)]
drawn = set(random.Random(args.seed).sample(others, min(len(others), args.min)))
sample = [i for i in others if i in drawn]
keep = set(must) | drawn
ids = [c.id for c in found.claims if c.id in keep]
doc = {"schema": "okf-wiki/claim-sample/1", "seed": args.seed, "min": args.min, "of_all": bool(args.of_all),
"claims_sha256": _sha(claims_path), "judged_sha256": judged_sha,
"part_manifest_sha256": manifest_sha,
"counts": {"claims": len(found.claims), "judged": len(found.claims) - len(open_), "must_read": len(must),
"others": len(others), "sample": len(sample)},
"must_read": must, "sample": sample, "ids": ids,
"ids_sha256": hashlib.sha256("\n".join(ids).encode("utf-8")).hexdigest(),
"error_bound_at_zero": error_bound(0, len(sample))}
Path(args.out).write_text(_dump(doc), encoding="utf-8")
print(f"sample (seed {args.seed}): {len(found.claims)} claims, {len(found.claims) - len(open_)} judged, "
f"{len(must)} must be read, {len(sample)} drawn of {len(others)}; read {len(ids)}")
print(f"0 not supported in the sample would bound the error at {error_bound(0, len(sample)):.2%} "
"(one-sided Clopper-Pearson, 95 %)")
return 0
def cmd_score(args: argparse.Namespace) -> int:
try:
found = load_claims(Path(args.claims))
verdicts = json.loads(Path(args.verdicts).read_text(encoding="utf-8"))
ids = _ids(Path(args.ids)) if args.ids else None
sample = json.loads(Path(args.ids).read_text(encoding="utf-8")).get("sample") if args.ids else None
except (OSError, ValueError, KeyError, TypeError) as exc:
print(f"score cannot run: {exc}", file=sys.stderr)
return 2
report = criterion_2(found, verdicts, ids)
_print("criterion 2, supported", report)
marks = ", ".join(f"{k} {a} of {b}" for k, (a, b) in report["not_supported_per_mark"].items())
print(f"not supported per provenance mark: {marks or 'none'}")
if isinstance(sample, list):
k = len(set(report["not_supported"]) & set(sample)) + len(set(sample) - {c.id for c in found.claims})
print(f"{k} not supported in a sample of {len(sample)}: upper bound {error_bound(k, len(sample)):.2%} "
"(one-sided Clopper-Pearson, 95 %)")
if report["not_supported"]:
print("not supported: " + " ".join(report["not_supported"]))
x, n = report["total"]
return 0 if n and x == n else 1
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
ls = sub.add_parser("list", help="list the claims and count criterion 1")
ls.add_argument("bundle")
ls.add_argument("sources")
ls.add_argument("--out", required=True)
ls.add_argument("--pairs")
ca = sub.add_parser("calls", help="lay the claims out in calls of at most ten per page")
ca.add_argument("--claims", required=True)
ca.add_argument("--pairs", required=True)
ca.add_argument("--out", required=True)
ca.add_argument("--ids")
ca.add_argument("--size", type=int, default=CALL_SIZE,
help=f"at most this many claims per call, 1 to {CALL_SIZE} (default {CALL_SIZE})")
sa = sub.add_parser("sample", help="the claims to read: the part, inferred, merged and a seeded draw")
sa.add_argument("--claims", required=True)
sa.add_argument("--judged")
sa.add_argument("--part-root")
sa.add_argument("--of-all", action="store_true", help="draw among every claim and add nothing")
sa.add_argument("--seed", type=int, required=True)
sa.add_argument("--min", type=int, required=True)
sa.add_argument("--out", required=True)
sc = sub.add_parser("score", help="count criterion 2 from a reader's verdicts")
sc.add_argument("--claims", required=True)
sc.add_argument("--verdicts", required=True)
sc.add_argument("--ids")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
commands = {"list": cmd_list, "calls": cmd_calls, "sample": cmd_sample, "score": cmd_score}
if args.command in ("list", "calls", "sample"):
try:
return commands[args.command](args)
except (OSError, ValueError, KeyError, TypeError) as exc:
print(f"claims cannot run: {exc}", file=sys.stderr)
return 2
return commands[args.command](args)
if __name__ == "__main__":
sys.exit(main())

121
okfwiki/common.py Normal file
View file

@ -0,0 +1,121 @@
"""Helpers shared by the okf-wiki tools. No model call (CLAUDE.md rule 1)."""
from __future__ import annotations
import re
import unicodedata
from pathlib import Path
from typing import Any
# A release note's version heading: "1.2.3" or "v1.2.3", at least three parts.
VERSION_HEADING = re.compile(r"^v?(\d+(?:\.\d+){2,})$")
# A line that is only one HTML or MDX tag, such as <Note> or </Steps>: layout, not content. The tag name is letters,
# digits, ".", "_" or "-", followed by whitespace, "/" or ">", so a link in angle brackets (<https://...>) is not a tag.
HTML_TAG_ONLY = re.compile(r"^\s*</?[A-Za-z][A-Za-z0-9._-]*(?:\s[^<>]*)?/?>\s*$")
class RunError(Exception):
"""The command cannot run as asked; nothing was written. A command reports it with exit 2."""
def write_new(path: Path, text: str) -> None:
"""Write text to a file that does not exist yet; an existing file is never overwritten."""
if path.exists():
raise RunError(f"{path} exists; refusing to overwrite")
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
def jaccard(a: set[str], b: set[str]) -> float:
return len(a & b) / len(a | b) if a | b else 0.0
# Letters that NFKD does not split into an ASCII letter and a mark, spelled out as Norwegian and German write them.
SPELLED = {"æ": "ae", "ø": "oe", "å": "aa", "ß": "ss", "œ": "oe", "þ": "th", "ð": "d", "ł": "l", "đ": "d"}
def ascii_fold(text: str) -> str:
"""Lower-case text with every letter spelled in ASCII where it can be: æ ø å as ae oe aa, an accent dropped
(é -> e); what has no ASCII spelling is left as it is."""
out = []
for ch in unicodedata.normalize("NFC", text).lower():
if ch in SPELLED:
out.append(SPELLED[ch])
else:
out.append("".join(c for c in unicodedata.normalize("NFKD", ch) if not unicodedata.combining(c)))
return "".join(out)
def slug(text: str) -> str:
"""Lower-case letters and digits joined by single hyphens, at most 60 characters; "x" when nothing is left.
Letters outside ASCII are spelled out first (ascii_fold), so "Årlig" is "aarlig", not "rlig"."""
return re.sub(r"[^a-z0-9]+", "-", ascii_fold(text)).strip("-")[:60] or "x"
def place_words(locator: dict[str, Any]) -> str:
"""Where a quote stands, as a reader names it: page 3, paragraph 4, sheet Plan, cell B7, slide 2 (notes), or line N
when the source is a text file."""
if "page" in locator:
return f"page {locator['page']}"
if "paragraph" in locator:
return f"paragraph {locator['paragraph']}"
if "cell" in locator:
cell = str(locator["cell"])
return f"sheet {locator.get('sheet', '')}, {'cells' if ':' in cell else 'cell'} {cell}"
if "slide" in locator:
return f"slide {locator['slide']}" + (" (notes)" if locator.get("notes") else "")
return f"line {locator.get('line')}"
def place_at(place: dict[str, Any] | None, line: str, quote: str) -> dict[str, Any]:
"""The place in the original of a quote on a line of a frozen copy, from that line's entry in K1 `places`. A
spreadsheet row's entry lists its cells with the offset each starts at, so the quote gets the cell, or the range
of cells, it covers; a quote not on the line gets the whole row. Any other entry is the place as it stands."""
if not place:
return {}
if "cells" not in place:
return dict(place)
cells = [(str(ref), int(at)) for ref, at in place["cells"]]
start = line.find(quote) if quote else -1
end = start + len(quote)
if start < 0:
covered = [ref for ref, _ in cells]
else:
bounds = [at for _, at in cells[1:]] + [len(line) + 1]
covered = [ref for (ref, at), stop in zip(cells, bounds) if at < end and stop > start]
cell = covered[0] if len(covered) == 1 else f"{covered[0]}:{covered[-1]}"
return {"sheet": place["sheet"], "cell": cell}
def place_fits(cited: dict[str, Any], frozen: dict[str, Any] | None) -> bool:
"""Whether a cited place (an x-locator without its line) is the frozen line's place: for a spreadsheet row, the
same sheet and a cell, or a range from one cell to a later one, of that row; otherwise the same place."""
if not frozen or "cells" not in frozen:
return cited == (frozen or {})
refs = [str(ref) for ref, _ in frozen["cells"]]
if set(cited) != {"sheet", "cell"} or cited["sheet"] != frozen["sheet"] or not isinstance(cited["cell"], str):
return False
ends = cited["cell"].split(":")
if not 1 <= len(ends) <= 2 or not all(e in refs for e in ends):
return False
return refs.index(ends[0]) <= refs.index(ends[-1])
# Common words of each language the view can name (no word in both lists): a text's language is the one whose words
# it uses clearly more, as a BCP 47 tag for the HTML lang attribute.
LANGUAGE_WORDS = {
"no": frozenset("og er det som på av til en et ikke med har den de seg var kan om ved fra også eller skal vil"
" når så etter må blir være hvor hva dette disse andre mer bare mellom hver".split()),
"en": frozenset("the and is of to that it not with has they was can about from also or shall will when"
" after must be where what this these other more only between under over each are".split()),
}
LANGUAGE_MIN = 5 # common words a text must have before its language is told
def language(text: str) -> str | None:
"""The language of text ("no" or "en"), or None when it has too few common words or no clear winner."""
words = re.findall(r"[^\W\d_]+", text.lower())
counts = {lang: sum(1 for w in words if w in common) for lang, common in LANGUAGE_WORDS.items()}
best = max(counts, key=lambda k: counts[k])
rest = max(v for k, v in counts.items() if k != best)
return best if counts[best] >= LANGUAGE_MIN and counts[best] >= 2 * rest else None

661
okfwiki/cost.py Normal file
View file

@ -0,0 +1,661 @@
"""Token cost of a v1 run from its sessions' transcripts: cost lines, stop checks and the isolation count. No model.
Usage:
python3 -m okfwiki.cost harvest --transcripts DIR --okter FILE --work DIR --out kost.json [--packs FILE] [--margin M]
python3 -m okfwiki.cost stop --lines L[,L] --cap N --unit blocks|passes|claims --done D --total T --kost FILE
[--blocks FILE] [--batches-done B --batches-total N] [--next B [--next-chars C]]
[--margin M]
[--name NAME]
python3 -m okfwiki.cost summarize --kost FILE [--cap-group NAME=L[,L]:CAP]...
python3 -m okfwiki.cost isolation --transcripts DIR --run R --tasks DIR --out isolation.json
[--line read|diagnose|repair]
harvest Reads the sessions listed in okter.txt ("<session-id> <kind>" per line, kind build, read, diagnose, merge or
repair) under
the transcripts directory (~/.claude/projects/<slug>/): each session's main transcript is orchestration, on
the line orchestration-<kind>, with the input side of every turn kept by turn number, the first turn that
starts helpers (an Agent, Task or Workflow call) and the tokens of the turns before it; every subagent
transcript under <session>/subagents/ or, for a workflow's agents, <session>/subagents/workflows/<id>/ (the
agent() label is the description) is attributed by its description "V1-<run>-<line>-<unit>", line pack (kept
as build), link, read, diagnose, merge (step 18a's pair helper, order 29), repair (the repair helper, order 34)
or calib. Any other description is an unattributed run: it is listed and the exit is 1,
because a model-run session spawns no helper agents. Tokens are counted as the trial did: per message id
once, at its largest value; an output never recorded is derived from the next message's input growth (a
lower bound). With --packs, a pack log with a gap (a logged pack with no build run, or a build run for a pack
the log lacks) cannot run. The margin is max(0.10, measured), 0.10 when nothing is measured, because the
measurement is itself a lower bound. Recorded cap groups and stop rows survive a new harvest.
stop The stop check after a pack or a pass. Measured = the sum of the given lines (calib is reported, never
capped). Flat projection = measured / done x total. --unit blocks also fits cost per block against blocks
read before (a + b*R) over the packs in --blocks (a pack log with "pack" and "blocks"), orchestration spread
over the packs in proportion to their attributed cost, and projects a*T + b*T*T/2, once at least four packs
are measured (the operator 11.10.2026); before that the flat projection alone. --unit claims scales only the
helper lines per claim: an orchestration line counts the session's part before its first helper once and the
rest per batch (a message that starts helpers, or a phase of a workflow), measured / --batches-done x
--batches-total, both required with an orchestration line. STOP when the projection x (1 + margin) exceeds
the cap. With --next B (--unit blocks only) the projection is recorded and printed but does not stop:
STOP when (measured + measured / done x B) x (1 + margin) exceeds the cap, B being the blocks of the pack
the model works on before the next check (the operator 12.10.2026: a cost line stops on measured cost plus a
margin, the projection is reported only; order 22 session A3: plus the next pack, so the cap is not passed).
With --next-chars C as well, the next pack's share is by characters, not blocks (motor B17: a block of a
folder's PDF or spreadsheet holds about ten times the characters of a docs block, and the cost follows the
characters): measured / the `block_chars` of the packs with a compiler run here x C, ROUND2_PER_CHAR before
the first; a log whose built packs have no `block_chars` keeps the rule by blocks.
The cap group and the stop row are recorded in the kost file.
Exit 0 continue, 1 STOP, 2 cannot run; a caller treats every non-zero exit as STOP.
summarize Per line "<line>: <lower bound> + margin <m> = <sum>"; per cap group (recorded by stop, or given with
--cap-group) "<name>: <lower bound> + margin <m> = <sum> of <cap>"; orchestration tokens per turn, per pack
and the input per turn fitted against the turn number. Exit 1 when a group is over its cap.
isolation Reader agents (or, with --line diagnose, diagnosis agents) are one per task, described
"V1-<run>-<line>-<task-id>", each with the task file <tasks>/<task-id>.md, started from a session or from a workflow (the same two places as harvest). Every
Read call of another path is a break. 0 breaks over at least one agent = exit 0. With --line repair (order
34), each repair helper "V1-<run>-repair-<task-id>" may read exactly the parts and write exactly the one
answer path of <tasks>/<task-id>.parts.json (krepair task's part list); every other tool call, a tool other
than Read and Write among them, is a break (CLAUDE.md: the breach count reads every tool call), and so is a
part never read and an answer written other than once (order 39). A helper
whose task has no part list cannot run (exit 2).
"""
from __future__ import annotations
import argparse
import json
import math
import re
import sys
from collections.abc import Mapping
from fractions import Fraction
from pathlib import Path
from typing import Any
from okfwiki.common import RunError
from okfwiki.transcript import USAGE_FIELDS, read_transcript
SCHEMA = "okf-wiki/cost/1"
ISOLATION_SCHEMA = "okf-wiki/isolation/1"
LINES = ("build", "link", "orchestration-build", "read", "orchestration-read", "diagnose", "orchestration-diagnose",
"merge", "orchestration-merge", "repair", "orchestration-repair", "calib")
UNCAPPED = ("calib",)
LINE_OF = {"pack": "build", "link": "link", "read": "read", "diagnose": "diagnose", "merge": "merge",
"repair": "repair", "calib": "calib"}
KINDS = ("build", "read", "diagnose", "merge", "repair")
UNITS = ("blocks", "passes", "claims")
ISOLATED = ("read", "diagnose", "repair")
PREFIX = re.compile(r"^V1-(.+?)-(pack|link|read|diagnose|merge|repair|calib)-(\S+)$")
PACK_UNIT = re.compile(r"^p(\d+)(?:-|$)")
MARGIN_FLOOR = 0.10
CATEGORIES = {"input_tokens": "input", "cache_read_input_tokens": "cache_read",
"cache_creation_input_tokens": "cache_write", "output_tokens": "output"}
INPUT_SIDE = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens")
CHARS_PER_TOKEN_LOW = 2.5 # injected text counted high, so a derived output is a lower bound
SPAWNS = ("Agent", "Task", "Workflow") # the tool calls that start helpers or a workflow's agents
FIT_FROM_PACKS = 4 # the operator 11.10.2026: the fitted curve counts from pack 4
ROUND2_PER_BLOCK = 3_259 # round 2 of the first part, 14 741 606 over 4 523 blocks (order 25 session C): the next pack's
# share before a ledger's first compiler run (order 27 A3)
ROUND2_PER_CHAR = Fraction(14_741_606, 1_111_796) # the same cost over the 1 111 796 characters of round 2's blocks
# (about 13.3 per character): the share by characters before the first compiler run (B17)
def margin(measured: float | None) -> float:
"""The margin on a lower-bound measurement: the measured one, never below the 10 % floor."""
return MARGIN_FLOOR if measured is None else max(MARGIN_FLOOR, measured)
def sessions(okter: Path) -> list[tuple[str, str]]:
"""(session id, kind) per line of okter.txt, in order, each id once."""
out: list[tuple[str, str]] = []
for line in okter.read_text(encoding="utf-8").splitlines():
parts = line.split()
if not parts:
continue
if len(parts) != 2 or parts[1] not in KINDS:
raise RunError(f"okter line {line!r} is not '<session-id> <kind>' with kind in {', '.join(KINDS)}")
known = dict(out)
if parts[0] in known and known[parts[0]] != parts[1]:
raise RunError(f"session {parts[0]} is listed as both {known[parts[0]]} and {parts[1]}")
if parts[0] not in known:
out.append((parts[0], parts[1]))
return out
def _result_chars(message: dict[str, Any]) -> int:
content = message.get("content")
if not isinstance(content, list):
return 0
total = 0
for item in content:
if not (isinstance(item, dict) and item.get("type") == "tool_result"):
continue
inner = item.get("content")
if isinstance(inner, str):
total += len(inner)
elif isinstance(inner, list):
total += sum(len(str(x.get("text", ""))) for x in inner if isinstance(x, dict))
return total
def usage(path: Path, per_turn: bool = False) -> dict[str, Any]:
"""Tokens of one transcript, ported from the trial. A message whose records all lack a stop reason never got its
final usage written; when another message follows, its output is the growth of the next message's input side
less the text injected between them (counted high at CHARS_PER_TOKEN_LOW), a lower bound. With per_turn, the
input side of every turn is kept with its turn number."""
per: dict[str, dict[str, int]] = {}
order: list[str] = []
complete: set[str] = set()
spawning: set[str] = set()
between: dict[str, int] = {}
last_stop: str | None = None
for line in read_transcript(path):
message = line.get("message")
if line.get("type") == "attachment" and order:
between[order[-1]] = between.get(order[-1], 0) + len(str(line.get("rendered") or ""))
continue
if not isinstance(message, dict):
continue
if line.get("type") == "user" and order:
between[order[-1]] = between.get(order[-1], 0) + _result_chars(message)
continue
if line.get("type") != "assistant":
continue
mid = str(message.get("id", ""))
if mid not in per:
order.append(mid)
seen = per.setdefault(mid, {})
fields = message.get("usage") or {}
for f in USAGE_FIELDS:
value = fields.get(f)
if isinstance(value, int):
seen[f] = max(seen.get(f, 0), value)
content = message.get("content")
if isinstance(content, list) and any(isinstance(b, dict) and b.get("type") == "tool_use"
and b.get("name") in SPAWNS for b in content):
spawning.add(mid)
stop = message.get("stop_reason")
last_stop = stop if isinstance(stop, str) else None
if isinstance(stop, str):
complete.add(mid)
out: dict[str, Any] = {CATEGORIES[f]: sum(v.get(f, 0) for v in per.values()) for f in USAGE_FIELDS}
out["output_recorded"] = out["output"]
derived = unrecorded = 0
for i, mid in enumerate(order):
if mid in complete:
continue
if i + 1 == len(order):
unrecorded += 1
continue
growth = (sum(per[order[i + 1]].get(f, 0) for f in INPUT_SIDE) - sum(per[mid].get(f, 0) for f in INPUT_SIDE)
- int(between.get(mid, 0) / CHARS_PER_TOKEN_LOW))
recorded = per[mid].get("output_tokens", 0)
if growth > recorded:
out["output"] += growth - recorded
derived += 1
out["output_derived_messages"] = derived
out["output_unrecorded"] = unrecorded
out["total"] = sum(out[c] for c in CATEGORIES.values())
out["turns"] = len(order)
first = per[order[0]] if order else {}
out["startup"] = sum(first.get(f, 0) for f in INPUT_SIDE)
out["stop_reason"] = last_stop
if per_turn:
out["per_turn"] = [{"turn": i, "input": sum(per[mid].get(f, 0) for f in INPUT_SIDE)}
for i, mid in enumerate(order, 1)]
spawn_turn = next((i for i, mid in enumerate(order, 1) if mid in spawning), None)
before = order if spawn_turn is None else order[:spawn_turn - 1]
out["first_helper_turn"] = spawn_turn
out["before_first_helper"] = sum(per[mid].get(f, 0) for mid in before for f in USAGE_FIELDS)
return out
def pack_of(unit: str) -> int | None:
m = PACK_UNIT.match(unit)
return int(m.group(1)) if m else None
def line_totals(runs: list[dict[str, Any]], orchestration: Mapping[str, dict[str, Any]]) -> dict[str, dict[str, int]]:
out: dict[str, dict[str, int]] = {}
for name in LINES:
out[name] = {c: 0 for c in (*CATEGORIES.values(), "total", "turns", "agents")}
rows = [(r["line"], r) for r in runs] + [(f"orchestration-{o['kind']}", o) for o in orchestration.values()]
for name, row in rows:
for c in (*CATEGORIES.values(), "total", "turns"):
out[name][c] += int(row[c])
out[name]["agents"] += 1
return out
def agent_transcripts(subagents: Path, suffix: str) -> list[Path]:
"""The agent files of one session: its own helpers and every workflow's (measured 09.10, Claude Code 2.1.295)."""
return sorted([*subagents.glob(f"agent-*.{suffix}"), *subagents.glob(f"workflows/*/agent-*.{suffix}")])
def harvest(transcripts: Path, okter: list[tuple[str, str]]) -> dict[str, Any]:
runs: list[dict[str, Any]] = []
unattributed: list[dict[str, Any]] = []
orchestration: dict[str, dict[str, Any]] = {}
for sid, kind in okter:
top = transcripts / f"{sid}.jsonl"
if not top.is_file(): # order 27 A3: a wrong transcripts folder would give 0 for a whole ledger without a word
raise RunError(f"session {sid} in okter has no transcript {top}")
orchestration[sid] = {"kind": kind, **usage(top, per_turn=True)}
for path in agent_transcripts(transcripts / sid / "subagents", "jsonl"):
meta_path = path.with_name(path.stem + ".meta.json")
meta = json.loads(meta_path.read_text(encoding="utf-8")) if meta_path.is_file() else {}
row = {"session": sid, "agent": path.stem, "agentType": meta.get("agentType"),
"description": str(meta.get("description", "")), **usage(path)}
m = PREFIX.match(row["description"])
if m is None:
unattributed.append(row)
continue
row["run"], row["line"], row["unit"] = m.group(1), LINE_OF[m.group(2)], m.group(3)
row["pack"] = pack_of(m.group(3))
runs.append(row)
return {"sessions": [{"id": sid, "kind": kind} for sid, kind in okter], "runs": runs,
"unattributed": unattributed, "orchestration": orchestration, "lines": line_totals(runs, orchestration)}
def check_packs(runs: list[dict[str, Any]], packs_log: dict[str, Any]) -> None:
logged = sorted(int(p["pack"]) for p in packs_log["packs"])
if logged != list(range(1, len(logged) + 1)):
raise RunError(f"the pack log is not packs 1..{len(logged)}: {logged}")
built = {int(r["pack"]) for r in runs if r["line"] == "build" and r["pack"] is not None}
missing, extra = sorted(set(logged) - built), sorted(built - set(logged))
if missing or extra:
raise RunError(f"pack log gap: logged without a build run {missing}, built but not logged {extra}")
def allocate(pack_costs: Mapping[int, float], packs: list[list[str]], blocks_doc: dict[str, Any]) -> dict[str, float]:
"""Tokens per source: each pack's cost split over its sources by their share of the pack's blocks."""
count: dict[str, int] = {}
for b in blocks_doc["blocks"]:
count[b["file"]] = count.get(b["file"], 0) + 1
per: dict[str, float] = {}
for n, names in enumerate(packs, 1):
if n not in pack_costs:
continue
total = sum(count.get(x, 0) for x in names) or 1
for x in names:
per[x] = pack_costs[n] * count.get(x, 0) / total
return per
def fit_line(points: list[tuple[float, float]]) -> tuple[float, float]:
"""Least squares y = a + b*x; one distinct x gives b = 0."""
xs, ys = [x for x, _ in points], [y for _, y in points]
mx, my = sum(xs) / len(xs), sum(ys) / len(ys)
if len(set(xs)) == 1:
return my, 0.0
b = sum((x - mx) * (y - my) for x, y in points) / sum((x - mx) ** 2 for x in xs)
return my - b * mx, b
def project(points: list[tuple[float, int, int]], total_blocks: int) -> dict[str, Any]:
"""points: (cost, blocks in the unit, blocks read before it). Cost per block a + b*R, summed to a*T + b*T*T/2."""
if not points:
return {"fit": None, "flat": None, "T": total_blocks}
a, b = fit_line([(float(r), c / bl) for c, bl, r in points])
fit: dict[str, Any] = {"a": a, "b": b, "note": "one point, no growth term" if b == 0 and len(points) == 1
else f"{len(points)} points, not validated"}
fit["value"] = a * total_blocks + b * total_blocks * total_blocks / 2
flat = sum(c for c, _, _ in points) / sum(bl for _, bl, _ in points) * total_blocks
return {"fit": fit, "flat": flat, "T": total_blocks}
def capped_lines(spec: str) -> list[str]:
names = [x.strip() for x in spec.split(",") if x.strip()]
for name in names:
if name not in LINES:
raise RunError(f"unknown cost line {name!r}; lines are {', '.join(LINES)}")
if name in UNCAPPED:
raise RunError(f"{name} is reported, never capped")
if not names:
raise RunError("no cost line given")
return names
def claims_projection(kost: dict[str, Any], lines: list[str], done: int, total: int,
batches: tuple[int, int] | None) -> tuple[Fraction, dict[str, Any]]:
"""Helper lines per claim; each orchestration line's part before its session's first helper once, the rest per
batch. Measured 11.10 in order 12: scaling the whole sum per claim projected 162.8 M where the reading cost 10.5 M,
because the session's fixed start was scaled with the claims."""
helpers = sum(int(kost["lines"][name]["total"]) for name in lines if not name.startswith("orchestration-"))
projected = Fraction(helpers) * total / done
before = after = 0
for name in (x for x in lines if x.startswith("orchestration-")):
kind = name.removeprefix("orchestration-")
line_before = sum(int(o["before_first_helper"]) for o in kost.get("orchestration", {}).values()
if o.get("kind") == kind)
before += line_before
after += int(kost["lines"][name]["total"]) - line_before
parts: dict[str, Any] = {"helpers": helpers}
if any(x.startswith("orchestration-") for x in lines):
if batches is None:
raise RunError("--unit claims with an orchestration line needs --batches-done and --batches-total")
b_done, b_total = batches
if b_done <= 0 or b_done > b_total:
raise ValueError(f"--batches-done {b_done} --batches-total {b_total}: need 0 < done <= total")
projected += before + Fraction(after) * b_total / b_done
parts.update({"before_first_helper": before, "after_first_helper": after, "batches_done": b_done,
"batches_total": b_total})
return projected, parts
def ledger_rate(kost: dict[str, Any], packs_log: dict[str, Any], measured: int) -> tuple[Fraction, int]:
"""(cost per block, the blocks it is over): the measured cost over the blocks of the packs with a compiler run in
this ledger, so packs built in another ledger (a continued collection) never thin it; ROUND2_PER_BLOCK before the
ledger's first compiler run (order 27 A3). A compiler run for a pack the log lacks cannot run."""
blocks = {int(p["pack"]): int(p["blocks"]) for p in packs_log["packs"]}
over = sum(blocks[n] for n in built_packs(kost, packs_log))
return (Fraction(measured, over) if over else Fraction(ROUND2_PER_BLOCK)), over
def built_packs(kost: dict[str, Any], packs_log: dict[str, Any]) -> set[int]:
"""The packs with a compiler run in this ledger. A compiler run for a pack the log lacks cannot run."""
logged = {int(p["pack"]) for p in packs_log["packs"]}
built: set[int] = set()
for r in kost.get("runs", []):
if r.get("line") == "build" and r.get("pack") is not None:
if int(r["pack"]) not in logged:
raise RunError(f"a compiler run is for pack {r['pack']}, which the pack log lacks")
built.add(int(r["pack"]))
return built
def ledger_chars_rate(kost: dict[str, Any], packs_log: dict[str, Any], measured: int) -> tuple[Fraction, int] | None:
"""(cost per character, the characters it is over), as ledger_rate by the characters of the packs' blocks
(`block_chars`), ROUND2_PER_CHAR before the ledger's first compiler run; None when a built pack has no
`block_chars` (a log written before blocks had characters), which keeps the rule by blocks (B17)."""
chars = {int(p["pack"]): p.get("block_chars") for p in packs_log["packs"]}
built = built_packs(kost, packs_log)
if any(not isinstance(chars[n], int) for n in built):
return None
over = sum(int(chars[n] or 0) for n in built)
return (Fraction(measured, over) if over else ROUND2_PER_CHAR), over
def stop_row(kost: dict[str, Any], lines: list[str], cap: int, unit: str, done: int, total: int,
packs_log: dict[str, Any] | None, used_margin: float,
batches: tuple[int, int] | None = None, following: int | None = None,
next_chars: int | None = None) -> dict[str, Any]:
if next_chars is not None and (following is None or next_chars < 0):
raise ValueError(f"--next-chars {next_chars}: the characters of the next pack, with --next only")
if done <= 0 or total <= 0 or done > total:
raise ValueError(f"--done {done} --total {total}: need 0 < done <= total")
measured = sum(int(kost["lines"][name]["total"]) for name in lines)
flat = Fraction(measured) * total / done
rate = Fraction(measured, done)
parts: dict[str, Any] = {}
if unit == "claims":
flat, parts = claims_projection(kost, lines, done, total, batches)
fit: float | None = None
if unit == "blocks":
if packs_log is None:
raise RunError("--unit blocks needs --blocks (a pack log with pack and blocks)")
rate, over = ledger_rate(kost, packs_log, measured)
flat = measured + rate * (total - done) # the blocks not yet done at this ledger's rate
parts.update(rate=float(rate), rate_blocks=over)
attributed: dict[int, int] = {}
for r in kost.get("runs", []):
if r.get("line") in lines and r.get("pack") is not None:
attributed[int(r["pack"])] = attributed.get(int(r["pack"]), 0) + int(r["total"])
if attributed:
scale = measured / sum(attributed.values())
points: list[tuple[float, int, int]] = []
read = 0
for p in sorted(packs_log["packs"], key=lambda p: int(p["pack"])):
n, blocks = int(p["pack"]), int(p["blocks"])
if n in attributed:
points.append((attributed[n] * scale, blocks, read))
read += blocks
fitted = project(points, total)["fit"] if len(points) >= FIT_FROM_PACKS else None
fit = float(fitted["value"]) if fitted else None
worst = max(flat, Fraction(fit)) if fit is not None else flat
with_margin = worst * (1 + Fraction(str(used_margin)))
row = {"lines": lines, "cap": cap, "unit": unit, "done": done, "total": total, "measured": measured,
"flat": float(flat), "fit": fit, "margin": used_margin, "projected_with_margin": math.ceil(with_margin),
"stop": with_margin > cap, **parts}
if following is not None:
if unit != "blocks" or following < 0:
raise ValueError(f"--next {following}: the blocks of the next pack, with --unit blocks only")
measured_next = (measured + rate * following) * (1 + Fraction(str(used_margin)))
row.update(rule="measured", next=following, measured_next_with_margin=math.ceil(measured_next),
stop=measured_next > cap)
by_chars = ledger_chars_rate(kost, packs_log, measured) if next_chars is not None and packs_log else None
if by_chars is not None:
rate_c, over_c = by_chars
measured_next = (measured + rate_c * int(next_chars or 0)) * (1 + Fraction(str(used_margin)))
row.update(rule="measured-chars", next_chars=next_chars, rate_chars=float(rate_c), rate_chars_over=over_c,
measured_next_with_margin=math.ceil(measured_next), stop=measured_next > cap)
return row
def with_margin(lower: int, used: float) -> int:
return math.ceil(Fraction(lower) * (1 + Fraction(str(used))))
def summarize(kost: dict[str, Any], extra_groups: Mapping[str, dict[str, Any]]) -> tuple[list[str], bool]:
used = float(kost["margin"]["used"])
out: list[str] = []
for name in LINES:
lower = int(kost["lines"].get(name, {}).get("total", 0)) # a kost file from before a line existed
note = " (reported, never capped)" if name in UNCAPPED else ""
out.append(f"{name}: {lower} + margin {used:.2f} = {with_margin(lower, used)}{note}")
within = True
for name, group in {**kost.get("caps", {}), **extra_groups}.items():
lower = sum(int(kost["lines"][x]["total"]) for x in group["lines"])
total = with_margin(lower, used)
within = within and total <= int(group["cap"])
out.append(f"{name}: {lower} + margin {used:.2f} = {total} of {group['cap']}")
packs = {int(r["pack"]) for r in kost.get("runs", []) if r.get("pack") is not None}
orchestration = kost.get("orchestration", {})
for kind in KINDS:
mine = [o for o in orchestration.values() if o.get("kind") == kind]
turns = sum(int(o["turns"]) for o in mine)
if not turns:
continue
tokens = int(kost["lines"].get(f"orchestration-{kind}", {}).get("total", 0))
per_pack = f"; per pack {tokens // len(packs)} over {len(packs)} packs" if packs else ""
out.append(f"orchestration-{kind} per turn: {tokens // turns} over {turns} turns{per_pack}")
points = [(float(t["turn"]), float(t["input"])) for o in mine for t in o.get("per_turn", [])]
if len({x for x, _ in points}) > 1:
a, b = fit_line(points)
out.append(f"orchestration-{kind} input per turn: {round(a)} + {round(b)} x turn ({len(points)} turns)")
return out, within
def parse_cap_group(text: str) -> tuple[str, dict[str, Any]]:
name, sep, rest = text.partition("=")
lines, sep2, cap = rest.rpartition(":")
if not (sep and sep2 and name.strip()):
raise RunError(f"--cap-group {text!r} is not NAME=LINES:CAP")
return name.strip(), {"lines": capped_lines(lines), "cap": int(cap)}
def part_list(tasks: Path, task: str) -> tuple[set[Path], Path]:
"""(the parts a repair helper may read, the one answer path it writes) from krepair task's part list."""
path = tasks / f"{task}.parts.json"
if not path.is_file():
raise RunError(f"repair task {task} has no part list {path}")
doc = load_json(path)
parts, answer = doc.get("parts"), doc.get("answer")
if not (isinstance(parts, list) and parts and all(isinstance(p, str) for p in parts) and isinstance(answer, str)):
raise RunError(f"{path} is not a part list with parts and one answer path")
return {(tasks / p).resolve() for p in parts}, (tasks / answer).resolve()
def repair_breaks(path: Path, reads: set[Path], answer: Path) -> tuple[int, int, list[str]]:
"""(reads, writes, breaks) of one repair helper: every tool call is checked, not only Read, and a part it never
read or an answer it wrote other than once is a break of its own (order 39)."""
n_reads = n_writes = 0
seen: set[Path] = set()
breaks: list[str] = []
for record in read_transcript(path) if path.is_file() else []:
message = record.get("message")
content = message.get("content") if isinstance(message, dict) else None
for block in content if isinstance(content, list) else []:
if not (isinstance(block, dict) and block.get("type") == "tool_use"):
continue
name = str(block.get("name"))
target = (block.get("input") or {}).get("file_path")
where = Path(str(target)).resolve() if isinstance(target, str) else None
if name == "Read" and where in reads:
n_reads += 1
seen.add(where)
elif name == "Write" and where == answer:
n_writes += 1
else:
breaks.append(f"{name} {target}" if isinstance(target, str) else name)
breaks.extend(f"part not read: {p}" for p in sorted(reads - seen))
if n_writes != 1:
breaks.append("answer not written" if n_writes == 0 else f"answer written {n_writes} times")
return n_reads, n_writes, breaks
def isolation(transcripts: Path, run: str, tasks: Path, line: str = "read") -> dict[str, Any]:
prefix = re.compile(rf"^V1-{re.escape(run)}-{re.escape(line)}-(\S+)$")
per_agent: list[dict[str, Any]] = []
for meta_path in sorted(p for d in transcripts.glob("*/subagents") for p in agent_transcripts(d, "meta.json")):
meta = json.loads(meta_path.read_text(encoding="utf-8"))
m = prefix.match(str(meta.get("description", "")))
if m is None:
continue
path = meta_path.with_name(meta_path.name.removesuffix(".meta.json") + ".jsonl")
if line == "repair":
n_reads, n_writes, found = repair_breaks(path, *part_list(tasks, m.group(1)))
per_agent.append({"agent": meta_path.name.removesuffix(".meta.json"), "claim": m.group(1),
"reads": n_reads, "writes": n_writes, "breaks": found})
continue
task = (tasks / f"{m.group(1)}.md").resolve()
reads: dict[str, str] = {}
for record in read_transcript(path) if path.is_file() else []:
message = record.get("message")
content = message.get("content") if isinstance(message, dict) else None
for block in content if isinstance(content, list) else []:
if isinstance(block, dict) and block.get("type") == "tool_use" and block.get("name") == "Read":
target = str((block.get("input") or {}).get("file_path", ""))
reads[str(block.get("id") or len(reads))] = target
breaks = [t for t in reads.values() if Path(t).resolve() != task]
per_agent.append({"agent": meta_path.name.removesuffix(".meta.json"), "claim": m.group(1),
"reads": len(reads), "breaks": breaks})
return {"schema": ISOLATION_SCHEMA, "run": run, "line": line, "tasks": str(tasks), "agents": len(per_agent),
"breaks": sum(len(a["breaks"]) for a in per_agent), "per_agent": per_agent}
def load_json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8"))
def write_json(path: Path, doc: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(doc, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
h = sub.add_parser("harvest")
h.add_argument("--transcripts", required=True)
h.add_argument("--okter", required=True)
h.add_argument("--work", required=True)
h.add_argument("--out", required=True)
h.add_argument("--packs")
h.add_argument("--margin", type=float)
s = sub.add_parser("stop")
s.add_argument("--lines", required=True)
s.add_argument("--cap", type=int, required=True)
s.add_argument("--unit", choices=UNITS, required=True)
s.add_argument("--done", type=int, required=True)
s.add_argument("--total", type=int, required=True)
s.add_argument("--kost", required=True)
s.add_argument("--blocks")
s.add_argument("--batches-done", type=int)
s.add_argument("--batches-total", type=int)
s.add_argument("--next", type=int)
s.add_argument("--next-chars", type=int)
s.add_argument("--margin", type=float)
s.add_argument("--name")
m = sub.add_parser("summarize")
m.add_argument("--kost", required=True)
m.add_argument("--cap-group", action="append", default=[])
i = sub.add_parser("isolation")
i.add_argument("--transcripts", required=True)
i.add_argument("--run", required=True)
i.add_argument("--tasks", required=True)
i.add_argument("--out", required=True)
i.add_argument("--line", choices=ISOLATED, default="read")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
if args.command == "harvest":
out = Path(args.out)
h = harvest(Path(args.transcripts), sessions(Path(args.okter)))
if args.packs:
check_packs(h["runs"], load_json(Path(args.packs)))
old = load_json(out) if out.is_file() else {}
doc = {"schema": SCHEMA, "work": str(args.work), **h,
"margin": {"measured": args.margin, "used": margin(args.margin)},
"caps": old.get("caps", {}), "stops": old.get("stops", {})}
write_json(out, doc)
print(f"harvest: {len(h['runs'])} attributed runs, {len(h['unattributed'])} unattributed, "
f"{len(h['orchestration'])} orchestrating sessions -> {out}")
for u in h["unattributed"]:
print(f" unattributed: {u['agent']} {u['description']!r}")
return 1 if h["unattributed"] else 0
if args.command == "stop":
kost_path = Path(args.kost)
kost = load_json(kost_path)
lines = capped_lines(args.lines)
used = margin(args.margin) if args.margin is not None else float(kost["margin"]["used"])
packs_log = load_json(Path(args.blocks)) if args.blocks else None
given = (args.batches_done, args.batches_total)
if (given[0] is None) != (given[1] is None):
raise RunError("--batches-done and --batches-total go together")
batches = None if given[0] is None else (int(given[0]), int(given[1]))
row = stop_row(kost, lines, args.cap, args.unit, args.done, args.total, packs_log, used, batches, args.next,
args.next_chars)
name = args.name or ",".join(lines)
kost.setdefault("caps", {})[name] = {"lines": lines, "cap": args.cap}
kost.setdefault("stops", {})[name] = row
write_json(kost_path, kost)
fit = f", fit {row['fit']:.0f}" if row["fit"] is not None else ""
if "batches_total" in row:
fit += (f" (helpers {row['helpers']} x {args.total}/{args.done}, before the first helper "
f"{row['before_first_helper']} once, after it {row['after_first_helper']} x "
f"{row['batches_total']}/{row['batches_done']} batches)")
if row.get("rule") == "measured-chars":
rate = (f"{row['measured']}/{row['rate_chars_over']} (the characters with a compiler run here)"
if row["rate_chars_over"] else
f"{float(ROUND2_PER_CHAR):.2f} (round 2's price per character; no compiler run here)")
print(f"{name}: measured {row['measured']} + next {row['next_chars']} characters x {rate}, "
f"x (1 + {used:.2f}) = {row['measured_next_with_margin']} of {args.cap}: "
f"{'STOP' if row['stop'] else 'continue'}; projection (reported only): flat {row['flat']:.0f}{fit}, "
f"x (1 + {used:.2f}) = {row['projected_with_margin']}")
elif "rule" in row:
rate = (f"{row['measured']}/{row['rate_blocks']} (the blocks with a compiler run here)"
if row["rate_blocks"] else f"{ROUND2_PER_BLOCK} (round 2's price per block; no compiler run here)")
print(f"{name}: measured {row['measured']} + next {row['next']} blocks x {rate}, "
f"x (1 + {used:.2f}) = {row['measured_next_with_margin']} of {args.cap}: "
f"{'STOP' if row['stop'] else 'continue'}; projection (reported only): flat {row['flat']:.0f}{fit}, "
f"x (1 + {used:.2f}) = {row['projected_with_margin']}")
else:
print(f"{name}: measured {row['measured']}, flat {row['flat']:.0f}{fit}, x (1 + {used:.2f}) = "
f"{row['projected_with_margin']} of {args.cap}: {'STOP' if row['stop'] else 'continue'}")
return 1 if row["stop"] else 0
if args.command == "summarize":
extra = dict(parse_cap_group(g) for g in args.cap_group)
text, within = summarize(load_json(Path(args.kost)), extra)
print("\n".join(text))
return 0 if within else 1
doc = isolation(Path(args.transcripts), args.run, Path(args.tasks), args.line)
write_json(Path(args.out), doc)
kind = {"read": "reader", "diagnose": "diagnosis", "repair": "repair"}[args.line]
print(f"isolation: {doc['breaks']} breaks over {doc['agents']} {kind} agents")
return 0 if doc["agents"] and doc["breaks"] == 0 else 1
except (RunError, OSError, json.JSONDecodeError, KeyError, ValueError, ZeroDivisionError, TypeError) as exc:
print(f"cost cannot run: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())

573
okfwiki/kassemble.py Normal file
View file

@ -0,0 +1,573 @@
"""K's assembly: concept pages from accepted fragments, by code. No model call (CLAUDE.md rule 1).
Usage:
python3 -m okfwiki.kassemble build --work DIR --sources DIR --out DIR [--links FILE] [--replace] [--at ISO]
One page per concept that at least one accepted fragment (WORK/K-work/accepted.json) enriches; a concept without a
fragment gets none. Docs fragments form the body in the order of their sources (pack, position, then line); the
fragments of a history source go under "## History": release notes in version order, then the pages K1 marks as
history (main_topic false, the weekly What's New pages) in week order. Every sentence ends in a footnote to its
fragment's source entry and, for a history source, to its label entry: for a release note the entry whose quote is the
bare version on the note's heading line, for a weekly page the entry titled with the week from the file name
(YYYY-wNN), whose quote is the page's first heading. A fragment's source entry quotes the whole line its quote
stands on, trimmed (`x-quote`), so the reader sees all the line says, and each context quote kpack placed (a
heading above it or another line of its section) is a source entry of its own, cited after it. Each fragment's
provenance mark is `x-provenance` on its source entries. SOURCES' K1 manifest (source-manifest.json) says which source is history; without it, release-* files are. A footnote label is keyed by its fragment (file,
span, quote line, quote, provenance), never by its place on the page, so one source's fragments can be replaced
without renumbering the others' (SPEC 5.1: "a stable `id` survives reordering").
A page's description is the first sentence of its first fragment, of the fragments that are its own when kmerge has
moved others in (a fragment marked `merged_from`), so the description follows the page's name. The catalogue's other
names for the concept (the aliases of its entry that are not the title, once, case set aside) stand on the page as
`x-aliases`, left out when there is none, and end its index line as " (also: A; B)".
Main pages and homes (accepted.json main_pages and homes, written by kpack from the compiler's and the guard's
answers): a page named as the main page of a docs source it cites carries `x-main-for`, the sorted list of those
sources as "/" + their K1 path, and the key is left out on every other page (contract K2). A main page lists the
pages whose home it is under "## Related concepts", one inline relative link per item, before the footnote
definitions; a home on a page that is no main page, or on itself, places nothing.
Code makes no other link: every inline link in a fragment's text, to a page or to the web, is reduced to its text, and
in-text links come only from --links, a links file of the link pass, placed by klink.place. A link, in-text or
related, whose target page no longer exists is dropped, and the pages that held one are listed; a record of the
--links file into or on a page that is gone is taken out of that file once the build is in (its accepted count
follows), so no later step places it again and it never counts as stale. `generated.at` is the content's last meaningful change (SPEC 5.2): a page
whose content is the same as in the existing output keeps its old `generated` and its bytes; any other page gets --at
(default now), written in UTC. index.md lists every page; log.md keeps every old line and gains one entry, newest
first (format choice d). Format choices (a)-(h) of PLAN.md section Rammer: sources[].resource starts with /, body links
are relative, no `verified` is written, quote and locator are x- fields.
The output is written to a staging folder beside --out and swapped in; the old --out is set aside beside it until the
new one is in, and comes back if the swap fails. An existing --out is refused unless --replace, and refused even then
when it holds anything this command does not write: a file other than index.md, log.md and pages whose generated.by is
this generator (a file browser's .DS_Store is tolerated and dropped). A marker left in WORK/K-work by an unfinished
kpack write stops the build. Exit 0 = built, 1 = built but some link records are stale (their sentence no longer
stands), 2 = could not run (also for an input the code did not foresee, with its traceback).
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import shutil
import sys
import tempfile
import traceback
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path, PurePosixPath
from typing import Any
import yaml
from okfwiki import audit, kblocks, klink
from okfwiki.common import RunError, place_at, place_words
from okfwiki.kblocks import headings_of, kilde_of, read_lines, version_key
from okfwiki.kpack import RESERVED_IDS, Fact, check_pending, read_index, read_json, source_facts, week_of
GENERATOR = "okf-wiki/kassemble"
LABEL_HEX = 10
SAFE_ID = re.compile(r"[a-z0-9][a-z0-9-]*")
RECORD_KEYS = {"pack": int, "position": int, "file": str, "start": int, "end": int, "section": str, "text": str,
"quote": str, "quote_line": int, "provenance": str, "concept_id": str}
LINK_KEYS = {"page": str, "sentence": str, "anchor": str, "at": int, "target": str}
MAIN_KEYS = {"source": str, "concept_id": str}
HOME_KEYS = {"source": str, "concept_id": str, "home": str}
RELATED = "## Related concepts"
TOLERATED = frozenset({".DS_Store"}) # folder metadata a file browser writes; dropped by the swap
@dataclass
class Page:
name: str
fm: dict[str, Any] # every key but `generated`
body: str # everything after the frontmatter block
@dataclass
class Report:
pages: int = 0
fragments: int = 0
new: list[str] = field(default_factory=list)
changed: list[str] = field(default_factory=list)
unchanged: list[str] = field(default_factory=list)
gone: list[str] = field(default_factory=list)
placed: int = 0
stale: int = 0
main_pages: int = 0
related: int = 0
dropped: dict[str, set[str]] = field(default_factory=dict) # page -> targets that are gone
unlinked: int = 0 # link records into a page that is gone, taken out of the links file
def frontmatter(data: dict[str, Any]) -> str:
"""A frontmatter block; PyYAML quotes any string that would load as another type (no, 1.10)."""
return "---\n" + yaml.safe_dump(data, sort_keys=False, allow_unicode=True, width=10_000) + "---\n"
def head(fm: dict[str, Any], at: str) -> str:
return frontmatter({**fm, "generated": {"by": GENERATOR, "at": at}})
def clean(text: str) -> str:
"""A fragment's text as body prose: no link (bundle or external) and no footnote mark of its own, on one line."""
text = audit.FOOTNOTE_REF.sub("", audit.sub_links(lambda m: m.group(1), text))
return re.sub(r"\s*\n\s*", " ", text).strip()
def footnoted(text: str, labels: list[str]) -> str:
if text and text[-1] not in ".!?":
text += "."
marks = "".join(f"[^{x}]" for x in labels)
out, last = [], 0
for _a, b in audit.sentence_spans(text):
out.append(text[last:b] + marks)
last = b
out.append(text[last:])
return "".join(out)
def label(r: dict[str, Any]) -> str:
key = "\0".join([r["file"], str(r["start"]), str(r["end"]), str(r["quote_line"]), r["quote"], r["provenance"]])
return "s" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:LABEL_HEX]
def context_label(file: str, line: int, provenance: str) -> str:
"""A context entry's label, keyed by its line and mark: fragments citing the same line share one entry."""
key = "\0".join(["context", file, str(line), provenance])
return "c" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:LABEL_HEX]
def version_label(version: str) -> str:
return "v" + version.replace(".", "-")
def version_line(lines: list[str], version: str, file: str) -> int:
"""The line that states the version: its heading, or the label of the component it is (kblocks.stated_versions)."""
n = kblocks.version_line(lines, version)
if n is None:
raise RunError(f"{file} has no heading or label for version {version}")
return n
def shown(line: str) -> str:
"""What a source entry quotes from a line: the whole line, trimmed, or a heading's text (order 22: a quote that is
a piece of its line hid from the reader what the claim drew on)."""
m = audit.HEADING.match(line)
return (m.group(2) or "").strip() if m else line.strip()
def one_line(text: str) -> str:
return " ".join(str(text).split())
def history_entry(file: str, lines: list[str], resource: str) -> dict[str, Any]:
"""The label entry of a history page without a version: titled with the week from its file name (or, failing
that, its first heading), quoting its first heading on that line."""
heads = headings_of(lines)
if not heads:
raise RunError(f"{file} is history but has no heading to quote for its label")
line, _level, text = heads[0]
week = week_of(file)
return {"id": "w" + week if week else "h-" + re.sub(r"[^a-z0-9]+", "-", file[:-3].lower()).strip("-"),
"resource": resource, "title": week or text, "x-locator": {"line": line}, "x-quote": text,
"x-provenance": "documented"}
def history_key(r: dict[str, Any]) -> tuple[Any, ...]:
if r.get("version"):
return 0, version_key(f"release-v{r['version']}.md")
week = week_of(str(r["file"]))
return (1, week) if week else (2, str(r["file"]))
def check_roles(raw: Any, key: str, keys: Mapping[str, type]) -> list[dict[str, Any]]:
"""accepted.json's main_pages or homes: absent is empty; a record without its keys cannot run."""
value = raw.get(key, []) if isinstance(raw, dict) else []
if not isinstance(value, list) or not all(isinstance(r, dict) and all(type(r.get(k)) is t for k, t in keys.items())
for r in value):
raise RunError(f"accepted.json: {key} is not a list of records with {', '.join(keys)}")
return value
def check_records(records: Any) -> list[dict[str, Any]]:
if not isinstance(records, list):
raise RunError("accepted.json has no fragments list")
for i, r in enumerate(records):
bad = [k for k, t in RECORD_KEYS.items() if not isinstance(r, dict) or type(r.get(k)) is not t]
if bad or not r["quote"].strip() or not (r.get("version") is None or isinstance(r["version"], str)):
raise RunError(f"accepted record {i} lacks {bad or ['a quote or a version string']}; every fragment "
f"carries a quote and a provenance mark (rule E)")
context = r.get("context", [])
if not (isinstance(context, list) and all(
isinstance(c, dict) and type(c.get("line")) is int and isinstance(c.get("quote"), str)
and c["quote"].strip() and isinstance(c.get("section"), str) for c in context)):
raise RunError(f"accepted record {i}: context is not a list of records with line, quote and section")
return records
def page_models(index: dict[str, Any], records: list[dict[str, Any]], kilde: Path,
facts: dict[str, Fact] | None = None, main_pages: list[dict[str, Any]] | None = None) -> dict[str, Page]:
facts = facts or {}
entries = {str(e["id"]): e for e in index["entries"]}
by_concept: dict[str, list[dict[str, Any]]] = {}
for r in records:
by_concept.setdefault(r["concept_id"], []).append(r)
cache: dict[str, list[str]] = {}
pages: dict[str, Page] = {}
for cid, items in sorted(by_concept.items()):
if cid not in entries:
raise RunError(f"concept {cid!r} has accepted fragments but no index entry")
if cid in RESERVED_IDS or not SAFE_ID.fullmatch(cid):
raise RunError(f"concept id {cid!r} cannot be a page name")
e = entries[cid]
if not (isinstance(e.get("title"), str) and e["title"].strip()):
raise RunError(f"the index entry of {cid!r} has no title")
title = one_line(e["title"])
items = sorted(items, key=lambda r: (r["pack"], r["position"], r["start"], r["end"]))
is_history = [bool(r.get("version")) or (r["file"] in facts and facts[r["file"]].history) for r in items]
body_items = [r for r, h in zip(items, is_history) if not h]
history = sorted((r for r, h in zip(items, is_history) if h), key=history_key)
sources: dict[str, dict[str, Any]] = {}
def add(entry: dict[str, Any]) -> str:
known = sources.setdefault(entry["id"], entry)
if known != entry:
raise RunError(f"two source entries on {cid}.md share the label {entry['id']}")
return str(entry["id"])
def source_lines(file: str) -> list[str]:
if file not in cache:
cache[file] = read_lines(kilde / file)
return cache[file]
def locator(file: str, line: int, text: str, quote: str) -> dict[str, Any]:
"""The line in the frozen copy, and for a PDF or Office source the place in the original (K1 places): in a
spreadsheet row, the cell or cells the quote covers (common.place_at); "" for a quote names the whole row."""
return {"line": line, **(place_at(facts[file].place(line), text, quote) if file in facts else {})}
def footnote(s: dict[str, Any]) -> str:
file = PurePosixPath(s["resource"]).name
original = facts[file].original if file in facts else None
return f"{PurePosixPath(original).name if original else file}, {place_words(s['x-locator'])}"
def cite(r: dict[str, Any]) -> list[str]:
src: dict[str, Any] = {"id": label(r), "resource": f"/kilde/{r['file']}"}
if r["section"]:
src["title"] = r["section"]
lines = source_lines(r["file"])
line = lines[r["quote_line"] - 1] if 0 < r["quote_line"] <= len(lines) else ""
quote = shown(line) if r["quote"] in line else r["quote"]
src.update({"x-locator": locator(r["file"], r["quote_line"], line, r["quote"]), "x-quote": quote, "x-span": [r["start"], r["end"]],
"x-provenance": r["provenance"]})
labels = [add(src)]
for c in r.get("context", []):
at = lines[c["line"] - 1] if 0 < c["line"] <= len(lines) else ""
ctx: dict[str, Any] = {"id": context_label(r["file"], c["line"], r["provenance"]),
"resource": f"/kilde/{r['file']}"}
if c["section"]:
ctx["title"] = c["section"]
ctx.update({"x-locator": locator(r["file"], c["line"], at, ""), "x-quote": shown(at) if c["quote"] in at else c["quote"],
"x-provenance": r["provenance"]})
labels.append(add(ctx))
if r.get("version"):
v = str(r["version"])
labels.append(add({"id": version_label(v), "resource": f"/kilde/{r['file']}", "title": v,
"x-locator": {"line": version_line(lines, v, r["file"])},
"x-quote": v, "x-provenance": "documented"}))
elif r["file"] in facts and facts[r["file"]].history:
labels.append(add(history_entry(r["file"], lines, f"/kilde/{r['file']}")))
return labels
lines = [f"# {title}", ""]
for r in body_items:
lines += [footnoted(clean(r["text"]), cite(r)), ""]
if history:
lines += ["## History", ""]
lines += [f"- {footnoted(clean(r['text']), cite(r))}" for r in history]
lines.append("")
lines += [f"[^{s['id']}]: {footnote(s)}" for s in sources.values()]
own = [r for r in [*body_items, *history] if not r.get("merged_from")] # the description follows the name
first = clean((own or body_items or history)[0]["text"])
spans = audit.sentence_spans(first)
fm: dict[str, Any] = {"type": "Concept", "title": title,
"description": first[spans[0][0]:spans[0][1]] if spans else title,
"sources": list(sources.values())}
names = other_names(title, e.get("aliases"))
if names:
fm["x-aliases"] = names
if e.get("kind") == "merge-candidate" and e.get("merge_of"):
fm["x-merge-candidate"] = str(e["merge_of"])
cited = {r["file"] for r in items}
mains = sorted({f"/kilde/{m['source']}" for m in main_pages or [] if m["concept_id"] == cid
and m["source"] in cited and m["source"] in facts and facts[m["source"]].main_topic})
if mains:
fm["x-main-for"] = mains
pages[f"{cid}.md"] = Page(f"{cid}.md", fm, "\n" + "\n".join(lines) + "\n")
return pages
def other_names(title: str, aliases: Any) -> list[str]:
"""The catalogue's other names for a concept, in its order: each alias that is not the title, once, with case set
aside."""
seen = {title.casefold()}
out: list[str] = []
for a in aliases if isinstance(aliases, list) else []:
name = one_line(str(a))
if name and name.casefold() not in seen:
seen.add(name.casefold())
out.append(name)
return out
def link_text(title: str) -> str:
return str(title).replace("[", "(").replace("]", ")")
def related(pages: dict[str, Page], homes: list[dict[str, Any]]) -> dict[str, list[str]]:
"""Per main page, the list items of the pages whose home it is: the home links the model chose, nothing else."""
out: dict[str, list[str]] = {}
for name, page in pages.items():
if not page.fm.get("x-main-for"):
continue
cid = name[:-3]
held = {str(h["concept_id"]) for h in homes if h["home"] == cid and h["concept_id"] != cid
and f"{h['concept_id']}.md" in pages}
items = sorted(held, key=lambda c: (str(pages[f"{c}.md"].fm["title"]).casefold(), c))
if items:
out[name] = [f"- [{link_text(pages[f'{c}.md'].fm['title'])}]({c}.md)" for c in items]
return out
def with_related(body: str, items: list[str]) -> str:
"""The body with the related-concepts list placed before its footnote definitions."""
if not items:
return body
lines = body.split("\n")
at = next((i for i, line in enumerate(lines) if audit.FOOTNOTE_DEF.match(line)), len(lines))
return "\n".join(lines[:at] + [RELATED, "", *items, ""] + lines[at:])
def gone_targets(old: Path, names: set[str], gone: set[str]) -> set[str]:
"""The pages gone since the last build that the old page `old` linked to."""
doc = audit.read_document(old, old.name)
targets = {PurePosixPath(t.split("#", 1)[0]).name for _, t in audit.body_links(doc) if audit.is_bundle_link(t)}
return targets & gone - names
def index_text(pages: dict[str, Page]) -> str:
"""The bundle-root index: one entry per page with its frontmatter description (SPEC 8), and after it the page's
other names (`x-aliases`) as " (also: A; B)", so a search of the index finds a concept by any of its names."""
out = ["---", 'okf_version: "0.2"', "---", "", "# Concepts", ""]
out += [f"- [{link_text(p.fm['title'])}]({name}) - {link_text(p.fm['description'])}"
+ (f" (also: {'; '.join(link_text(a) for a in p.fm['x-aliases'])})" if p.fm.get("x-aliases") else "")
for name, p in sorted(pages.items())]
return "\n".join(out) + "\n"
def log_text(old: str | None, at: str, entry: str) -> str:
"""log.md with one new entry, newest first under its date (SPEC 9); every old line stays as it was."""
day = at[:10]
if old is None:
return f"# Log\n\n## {day}\n{entry}\n"
lines = old.split("\n")
for i, raw in enumerate(lines):
m = audit.HEADING.match(raw)
text = (m.group(2) or "").strip() if m else ""
if m and len(m.group(1)) >= 2 and audit.DATE.match(text):
if text > day:
raise RunError(f"the build time {at} is older than the log's newest date {text}")
if text == day:
lines.insert(i + 1, entry)
else:
lines[i:i] = [f"## {day}", entry, ""]
return "\n".join(lines)
return old.rstrip("\n") + f"\n\n## {day}\n{entry}\n"
def generated_at(path: Path) -> str | None:
fm = audit.read_document(path, path.name).frontmatter or {}
gen = fm.get("generated")
at = gen.get("at") if isinstance(gen, dict) else None
return at if isinstance(at, str) else None
def written_here(path: Path) -> bool:
"""A concept page this command wrote: its generated.by names the generator."""
fm = audit.read_document(path, path.name).frontmatter or {}
gen = fm.get("generated")
return isinstance(gen, dict) and gen.get("by") == GENERATOR
def aside(out: Path) -> Path:
return out.with_name(f".{out.name}-old")
def check_out(out: Path, replace: bool) -> None:
if aside(out).exists():
raise RunError(f"{aside(out)} exists: an earlier swap did not finish; look at it before building again")
if not out.exists():
return
if not replace:
raise RunError(f"{out} exists; refusing to overwrite without --replace")
if not out.is_dir():
raise RunError(f"{out} is not a directory")
foreign = sorted(p.name for p in out.iterdir() if p.name not in TOLERATED and not (
p.is_file() and p.suffix == ".md" and (p.name in audit.RESERVED or written_here(p))))
if foreign or not (out / "log.md").is_file():
raise RunError(f"{out} holds what this command does not write ({foreign or 'no log.md'}); refusing to replace")
def check_links(path: Path) -> dict[str, Any]:
doc: dict[str, Any] = read_json(path)
records = doc.get("links") if isinstance(doc, dict) else None
if not isinstance(records, list) or not all(
isinstance(r, dict) and all(type(r.get(k)) is t for k, t in LINK_KEYS.items()) for r in records):
raise RunError(f"{path} is not a links file: a links list of records with {', '.join(LINK_KEYS)}")
return doc
def swap_in(new: Path, out: Path) -> None:
"""Put new in out's place. An old out is set aside beside it first and comes back if the swap fails."""
old = aside(out) if out.exists() else None
if old is not None:
out.rename(old)
try:
new.rename(out)
except BaseException:
if old is not None:
old.rename(out)
raise
if old is not None:
shutil.rmtree(old)
def utc(at: str) -> str:
if not audit.timestamp_ok(at):
raise RunError(f"--at {at!r} is not an ISO 8601 datetime with an explicit UTC offset (SPEC 5)")
return datetime.fromisoformat(at.replace("Z", "+00:00")).astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def build(work: Path, sources: Path, out: Path, links: Path | None = None, replace: bool = False,
at: str | None = None) -> Report:
check_pending(work)
at = utc(at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"))
kilde = kilde_of(sources)
accepted = work / "K-work" / "accepted.json"
if not accepted.is_file():
raise RunError(f"{accepted} does not exist; accept a pack first")
raw = read_json(accepted)
records = check_records(raw.get("fragments") if isinstance(raw, dict) else None)
main_pages, homes = check_roles(raw, "main_pages", MAIN_KEYS), check_roles(raw, "homes", HOME_KEYS)
facts = source_facts(sources, sorted({str(r["file"]) for r in records} | {str(m["source"]) for m in main_pages}))
pages = page_models(read_index(work), records, kilde, facts, main_pages)
lists = related(pages, homes)
check_out(out, replace)
link_doc = check_links(links) if links is not None else None
report = Report(pages=len(pages), fragments=len(records), main_pages=sum(bool(p.fm.get("x-main-for"))
for p in pages.values()),
related=sum(len(v) for v in lists.values()))
old_pages = {p.name for p in out.glob("*.md") if klink.is_concept(p.name)} if out.is_dir() else set()
gone = old_pages - set(pages)
out.parent.mkdir(parents=True, exist_ok=True)
stage = Path(tempfile.mkdtemp(prefix=f".{out.name}-", dir=out.parent))
try:
assembled = stage / "pages"
assembled.mkdir()
for name, page in pages.items():
(assembled / name).write_text(head(page.fm, at) + page.body, encoding="utf-8")
if link_doc is not None:
records = [{**r, "target": klink.page_rel(r["target"])} for r in link_doc["links"]]
live = [r for r in records if r["target"] in pages and r["page"] in pages]
for r in records:
if r["target"] not in pages and r["page"] in pages:
report.dropped.setdefault(r["page"], set()).add(r["target"])
report.unlinked = len(records) - len(live) # into or on a page that is gone
(stage / "links.json").write_text(json.dumps({**link_doc, "links": live}), encoding="utf-8")
report.placed, report.stale = klink.place(assembled, stage / "links.json", stage / "linked", replace=False)
assembled = stage / "linked"
final = stage / "final"
final.mkdir()
for name, page in pages.items():
_, body = klink.split_page((assembled / name).read_text(encoding="utf-8"))
body = with_related(body, lists.get(name, []))
old = out / name
old_text = old.read_text(encoding="utf-8") if old.is_file() else None
if old_text is not None:
lost = gone_targets(old, set(pages), gone)
if lost:
report.dropped.setdefault(name, set()).update(lost)
old_at = generated_at(old) if old_text is not None else None
if old_at is not None and head(page.fm, old_at) + body == old_text:
text = old_text
report.unchanged.append(name)
else:
text = head(page.fm, at) + body
(report.changed if old_text is not None else report.new).append(name)
(final / name).write_text(text, encoding="utf-8")
if out.exists():
report.gone = sorted(p.name for p in out.glob("*.md") if klink.is_concept(p.name) and p.name not in pages)
(final / "index.md").write_text(index_text(pages), encoding="utf-8")
dropped = sum(len(t) for t in report.dropped.values())
entry = (f"* **Build**: {GENERATOR}, {report.pages} pages from {report.fragments} accepted fragments: "
f"new {len(report.new)}, changed {len(report.changed)}, unchanged {len(report.unchanged)}, "
f"gone {len(report.gone)}" + (f"; main pages {report.main_pages}, related links {report.related}"
if report.main_pages else "")
+ (f"; links placed {report.placed}, stale {report.stale}" if links is not None else "")
+ (f"; dropped links to pages that are gone {dropped}" if dropped else "") + ".")
old_log = (out / "log.md").read_text(encoding="utf-8") if out.exists() else None
(final / "log.md").write_text(log_text(old_log, at, entry), encoding="utf-8")
swap_in(final, out)
if links is not None and link_doc is not None and report.unlinked:
kept = [r for r in link_doc["links"] if klink.page_rel(r["target"]) in pages and r["page"] in pages]
tmp = links.with_name(f".{links.name}.tmp")
tmp.write_text(klink.dump({**link_doc, "links": kept, "accepted": len(kept)}), encoding="utf-8")
tmp.replace(links) # after the swap, so a build that fails keeps every record
finally:
shutil.rmtree(stage, ignore_errors=True)
return report
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="python3 -m okfwiki.kassemble", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
b = sub.add_parser("build")
b.add_argument("--work", required=True)
b.add_argument("--sources", required=True)
b.add_argument("--out", required=True)
b.add_argument("--links")
b.add_argument("--replace", action="store_true")
b.add_argument("--at")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
r = build(Path(args.work), Path(args.sources), Path(args.out), Path(args.links) if args.links else None,
args.replace, args.at)
except (RunError, OSError, UnicodeDecodeError) as exc:
print(f"kassemble cannot run: {exc}", file=sys.stderr)
return 2
except Exception: # an input this code did not foresee: "could not run", never exit 1 (stale links)
traceback.print_exc()
return 2
print(f"kassemble: {r.pages} pages from {r.fragments} accepted fragments -> {args.out}")
print(f" new {len(r.new)}, changed {len(r.changed)}, unchanged {len(r.unchanged)}, gone {len(r.gone)}")
print(f" main pages {r.main_pages}, related links {r.related}")
if args.links:
print(f" links placed {r.placed}, stale {r.stale}")
if r.dropped:
print(f" dropped links to pages that are gone: {sum(len(t) for t in r.dropped.values())} "
f"({', '.join(sorted(r.dropped))})")
if r.unlinked:
print(f" removed from the links file: {r.unlinked} records into or on pages that are gone")
return 1 if r.stale else 0
if __name__ == "__main__":
sys.exit(main())

290
okfwiki/kblocks.py Normal file
View file

@ -0,0 +1,290 @@
"""K's sources in a fixed order, split into blocks with line spans. No model call (CLAUDE.md rule 1).
Usage:
python3 -m okfwiki.kblocks order SOURCES --out FILE
python3 -m okfwiki.kblocks blocks SOURCES --order FILE --out FILE
python3 -m okfwiki.kblocks total SOURCES
SOURCES is the folder that holds kilde/, the same raw-source root the auditor takes.
order every file under SOURCES/kilde/ in K's fixed order, one name per line: pages by name, then release notes
(release-vX.Y.Z.md) by version. Any other name is a page: a docs page (docs-*), or a document of a folder
(sources folder), .md or .txt alike.
blocks every block of every source in the order (schema okf-wiki/blocks/1): a paragraph, one list item with its
continuation lines, one table or one fenced code block, with its 1-based inclusive line span, its section (the
heading path), its characters (`chars`, every line with its newline) and, for a release note, its version.
Heading lines are section markers, not blocks, and a line that is only an HTML or MDX tag (<Note>, </Steps>)
never starts a block, so no block is only a tag. A "#" line
inside a component property's example string (example: `# Title ... `) is example text, not a heading.
total the number of blocks in every file under SOURCES/kilde/.
Lines are counted on the text split at "\\n", as the pack and the link pass count them. Exit 0 = done, 2 = could not
run; an existing output is never overwritten.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any
from okfwiki.audit import HEADING, LIST_ITEM, closes_fence, fence_marker
from okfwiki.common import HTML_TAG_ONLY, VERSION_HEADING, RunError, write_new
SCHEMA = "okf-wiki/blocks/1"
RELEASE = re.compile(r"^release-v(\d+(?:\.\d+)*)\.md$")
VERSION_LABEL = re.compile(r'^\s*<[A-Za-z][A-Za-z0-9._-]*\s[^<>]*?\blabel="v?(\d+(?:\.\d+){2,})"')
VERSION_BOLD = re.compile(r"^\*\*v?(\d+(?:\.\d+){2,})\*\*$")
# A component property whose value is a backtick string left open on its line (example: `# Title): the lines after it,
# up to the first with a backtick no backslash escapes, are example text, never a heading (order 14 findings C and D).
STRING_OPEN = re.compile(r"^\s*[A-Za-z_$][\w$]*\s*:\s*`")
TICK = re.compile(r"(?<!\\)`")
# Concept test 2 of the brief (Google's reference agent), plus the generic section names of docs pages.
META_STOP = frozenset({
"overview", "introduction", "getting started", "quickstart", "tutorial", "walkthrough", "release notes",
"changelog", "roadmap", "faq", "prerequisites", "next steps", "before you begin", "troubleshooting", "see also",
"related resources", "learn more",
})
@dataclass(frozen=True)
class Block:
file: str
start: int
end: int
kind: str # paragraph | item | table | code
section: str
version: str | None
chars: int = 0 # the characters of its lines, each with its newline: the unit the stop line prices (B17)
@dataclass(frozen=True)
class Heading:
file: str
line: int
level: int
text: str
def is_meta(title: str) -> bool:
return title.strip().lower() in META_STOP
def version_key(name: str) -> tuple[int, ...]:
m = RELEASE.match(name)
if m is None:
raise RunError(f"{name} is not a release note name")
return tuple(int(p) for p in m.group(1).split("."))
def order(names: list[str]) -> list[str]:
releases = [n for n in names if RELEASE.match(n)]
return sorted(n for n in names if not RELEASE.match(n)) + sorted(releases, key=version_key)
def source_files(kilde: Path) -> list[Path]:
"""Every file under kilde, in name order: .md and .txt alike, nothing dropped by its suffix."""
return sorted(p for p in kilde.iterdir() if p.is_file())
def read_lines(path: Path) -> list[str]:
return path.read_text(encoding="utf-8").split("\n")
def string_lines(lines: list[str]) -> set[int]:
"""The 1-based lines inside a component property's example string (STRING_OPEN), outside fenced code: from the line
after the one that opens it to the line with the backtick that closes it, both included."""
out: set[int] = set()
fence = "" # the open fence's marker run, or "" outside fenced code (the auditor's rule, CommonMark 4.5)
inside = False
for n, raw in enumerate(lines, 1):
if inside:
out.add(n)
inside = TICK.search(raw) is None
continue
f = fence_marker(raw)
if fence:
if closes_fence(f, raw, fence):
fence = ""
elif f:
fence = f.group(1)
elif STRING_OPEN.match(raw):
inside = len(TICK.findall(raw)) % 2 == 1
return out
def headings_of(lines: list[str]) -> list[tuple[int, int, str]]:
"""(line, level, text) of every heading outside fenced code and outside a component's example string."""
out: list[tuple[int, int, str]] = []
fence = ""
text_of_string = string_lines(lines)
for n, raw in enumerate(lines, 1):
if n in text_of_string:
continue
f = fence_marker(raw)
if fence:
if closes_fence(f, raw, fence):
fence = ""
continue
if f:
fence = f.group(1)
continue
m = HEADING.match(raw)
if m:
out.append((n, len(m.group(1)), (m.group(2) or "").strip()))
return out
def stated_versions(lines: list[str]) -> list[tuple[int, str]]:
"""(line, version) for every line that states a release note's version: a heading that is only the version, then
a component whose label is the version (<Update label="X.Y.Z" ...>: every release note of the copy of 08.10, 416
of 416, has this form and no heading; order 27 A6), then a line that is only the bold version (**X.Y.Z**: the
page's form after 20.10, 437 of 437 on 07.11)."""
found = [(n, m.group(1)) for n, _level, text in headings_of(lines) if (m := VERSION_HEADING.match(text))]
found += [(n, m.group(1)) for n, raw in enumerate(lines, 1) if (m := VERSION_LABEL.match(raw))]
return found + [(n, m.group(1)) for n, raw in enumerate(lines, 1) if (m := VERSION_BOLD.match(raw))]
def version_line(lines: list[str], version: str) -> int | None:
"""The first line that states `version`, or None."""
return next((n for n, v in stated_versions(lines) if v == version), None)
def blocks(path: Path) -> list[Block]:
lines = read_lines(path)
name = path.name
version: str | None = None
if RELEASE.match(name):
stated = stated_versions(lines)
version = stated[0][1] if stated else None
out: list[Block] = []
stack: list[tuple[int, str]] = []
kind: str | None = None
start = 0
fence = ""
text_of_string = string_lines(lines)
def close(end: int) -> None:
nonlocal kind
if kind is not None:
out.append(Block(name, start, end, kind, " › ".join(t for _, t in stack), version,
sum(len(line) + 1 for line in lines[start - 1:end])))
kind = None
for n, raw in enumerate(lines, 1):
f = fence_marker(raw)
if fence:
if closes_fence(f, raw, fence):
fence = ""
close(n)
continue
if f:
close(n - 1)
fence, kind, start = f.group(1), "code", n
continue
if not raw.strip():
close(n - 1)
continue
m = None if n in text_of_string else HEADING.match(raw)
if m:
close(n - 1)
level = len(m.group(1))
stack = [(lv, t) for lv, t in stack if lv < level] + [(level, (m.group(2) or "").strip())]
continue
if raw.lstrip().startswith("|"):
if kind != "table":
close(n - 1)
kind, start = "table", n
continue
if LIST_ITEM.match(raw):
close(n - 1)
kind, start = "item", n
continue
if kind in (None, "table"):
close(n - 1)
if HTML_TAG_ONLY.match(raw):
continue # a tag alone is layout; the block starts at the first line of content
kind, start = "paragraph", n
if fence is not None:
close(len(lines))
else:
last = len(lines)
while last > 0 and not lines[last - 1].strip():
last -= 1
close(last)
return out
def docs_headings(kilde: Path) -> list[Heading]:
"""The title and level 2-3 headings of every page under kilde (any file but a release note), in name order."""
out: list[Heading] = []
for path in (p for p in source_files(kilde) if not RELEASE.match(p.name)):
for n, level, text in headings_of(read_lines(path)):
if level <= 3 and text:
out.append(Heading(path.name, n, level, text))
return out
def blocks_document(kilde: Path, names: list[str]) -> dict[str, Any]:
found: list[dict[str, Any]] = []
for name in names:
path = kilde / name
if not path.is_file():
raise RunError(f"{path} is not a file")
found.extend(asdict(b) for b in blocks(path))
return {"schema": SCHEMA, "order": names, "blocks": found}
def kilde_of(sources: Path) -> Path:
kilde = sources / "kilde"
if not kilde.is_dir():
raise RunError(f"{kilde} is not a directory")
return kilde
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="python3 -m okfwiki.kblocks", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
o = sub.add_parser("order")
o.add_argument("sources")
o.add_argument("--out", required=True)
b = sub.add_parser("blocks")
b.add_argument("sources")
b.add_argument("--order", required=True)
b.add_argument("--out", required=True)
t = sub.add_parser("total")
t.add_argument("sources")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
kilde = kilde_of(Path(args.sources))
if args.command == "order":
fixed = order([p.name for p in source_files(kilde)])
write_new(Path(args.out), "".join(n + "\n" for n in fixed))
print(f"order: {len(fixed)} sources -> {args.out}")
elif args.command == "blocks":
names = Path(args.order).read_text(encoding="utf-8").split()
if Path(args.out).exists():
raise RunError(f"{args.out} exists; refusing to overwrite")
doc = blocks_document(kilde, names)
write_new(Path(args.out), json.dumps(doc, ensure_ascii=False, indent=1) + "\n")
print(f"blocks: {len(doc['blocks'])} in {len(names)} sources -> {args.out}")
else:
files = source_files(kilde)
print(f"total: {sum(len(blocks(p)) for p in files)} blocks in {len(files)} files of {kilde}")
except (RunError, OSError, UnicodeDecodeError) as exc:
print(f"kblocks cannot run: {exc}", file=sys.stderr)
return 2
return 0
if __name__ == "__main__":
sys.exit(main())

188
okfwiki/kguard.py Normal file
View file

@ -0,0 +1,188 @@
"""The guard after a repair: the pages without a home and the sources without a main page that the auditor reports in
a repaired collection are asked of the model again, and its answer is applied. No model call, no network, no program
started (CLAUDE.md rule 1). Order 35 A2b and B3.3 (the operator 22.10.2026, "Lag vakten som egen kommando
(Anbefalt)", PLAN.md § Rammer point 10): the guard of kpack run-after, used as a library and not changed.
Usage:
python3 -m okfwiki.kguard ask --work DIR --bundle WIKI --sources DIR --out DIR
python3 -m okfwiki.kguard apply --work DIR --asked FILE --answer FILE --bundle WIKI
- ask runs the frozen auditor in this process (kpack.run_audit) on --bundle against --sources, leaves out the retired
checks (kpack.RETIRED) and the findings the operator approved on his list (kpack.HOMELESS_APPROVED), and reads the
pages without a home and the sources without a main page as kpack does (klink.homeless). With no finding it writes
nothing and exits 0. A finding the guard cannot mend (a check outside klink.GUARDED) is named and stops: exit 5, and
nothing is written. Otherwise it writes the guard's task to --out with klink.guard_parts, the catalogue being
kpack.main_catalogue for the last pack of packs.json, and `asked.json` (okf-wiki/guard-asked/1: the pages, the
sources, the catalogue, the pack, the parts, the answer path, the collection's sha256, krepair.tree_sha256, and the sha256 of accepted.json), and
exits 4: run the home task. It never writes over a file that exists.
- apply does with the answer what kpack.apply_guard does: klink.read_guard against the pages that cite each asked
source (klink.guard_candidates), a main page per source in the source's place, a home per page with the source that
brings the page (kpack.first_files), then kpack.drop_dangling, written with kpack.write_state. It logs in its own
folder, WORK/K-guard/r35-<n>/guard.json, instead of packs.json. It refuses an asked task already applied (`applied`)
a --bundle that is not the collection asked (`stale-bundle`), and an accepted.json that changed since ask, as a
krepair fix between ask and apply changes it without changing the bundle (`stale-accepted`): exit 1, nothing
written.
Exit 0 done; 1 refused, one line per reason `refused: <code> <what>`; 2 cannot run; 4 the task is written (run it);
5 STOP: a finding the guard cannot mend.
"""
from __future__ import annotations
import argparse
import json
import sys
import traceback
from collections import Counter
from pathlib import Path
from typing import Any
from okfwiki import klink, kpack, krepair
from okfwiki.common import RunError
ASKED_SCHEMA = "okf-wiki/guard-asked/1"
LOG_SCHEMA = "okf-wiki/guard-log/1"
LOG_PREFIX = "r35-" # WORK/K-guard/r35-<n>/, one folder per applied answer (order 35 A2b)
LABEL = "after the repair"
EXIT_GUARD = kpack.EXIT_GUARD
EXIT_BREACH = kpack.EXIT_BREACH
def last_pack(work: Path) -> int:
packs = [int(r["pack"]) for r in kpack.pack_log(work)["packs"]]
if not packs:
raise RunError(f"{kpack.k_work(work) / 'packs.json'} lists no pack")
return max(packs)
def accepted_sha256(work: Path) -> str:
"""The sha256 of WORK's accepted.json, or "" when there is none: the accepted state a guard task was asked on."""
path = kpack.k_work(work) / "accepted.json"
return krepair.file_sha256(path) if path.is_file() else ""
def cmd_ask(args: argparse.Namespace) -> int:
work, bundle, sources, out = Path(args.work), Path(args.bundle), Path(args.sources), Path(args.out).resolve()
kpack.check_pending(work)
if out.exists() and any(out.iterdir()):
raise RunError(f"{out} holds files already; the guard's task is never written over what exists")
audit_doc = kpack.run_audit(bundle, sources)
approved = kpack.read_approved(work)
findings = [f for f in audit_doc["findings"] if isinstance(f, dict) and f.get("check") not in kpack.RETIRED
and not kpack.is_approved(f, approved)]
print(f"audit: {len(findings)} findings ("
+ (", ".join(f"{c} {k}" for c, k in sorted(Counter(str(f.get("check")) for f in findings).items()))
or "none") + ")")
if not findings:
print("guard: no page without a home, no source without a main page; no task")
return 0
cannot = [f for f in findings if f.get("check") not in klink.GUARDED]
if cannot:
print("STOP: a finding the guard cannot mend; the operator decides:")
for f in cannot:
print(f" {f.get('check')} {f.get('file') or ''}")
return EXIT_BREACH
pages, missing = klink.homeless({"findings": findings})
last = last_pack(work)
catalogue = [str(e["id"]) for e in kpack.main_catalogue(kpack.read_index(work), kpack.read_accepted(work), last)]
parts = klink.guard_parts(bundle, pages, missing, catalogue, LABEL)
paths = [out / f"part-{i}.md" for i in range(1, len(parts) + 1)]
answer = out / "answer.json"
asked = {"schema": ASKED_SCHEMA, "bundle": str(bundle.resolve()), "bundle_sha256": krepair.tree_sha256(bundle),
"accepted_sha256": accepted_sha256(work), "pack": last, "pages": pages, "sources": missing, "catalogue": catalogue,
"parts": [str(p) for p in paths], "answer": str(answer)}
out.mkdir(parents=True, exist_ok=True)
for path, text in [*zip(paths, parts), (out / "asked.json", kpack.dump(asked))]:
with path.open("x", encoding="utf-8") as fh:
fh.write(text)
print(f"guard: {len(pages)} pages without a home, {len(missing)} sources without a main page")
for p in paths:
print(f" part: {p}")
print(f" answer: {answer}")
return EXIT_GUARD
def logs(work: Path) -> list[Path]:
return sorted((work / "K-guard").glob(f"{LOG_PREFIX}*/guard.json"))
def cmd_apply(args: argparse.Namespace) -> int:
work, bundle, asked_path, answer = Path(args.work), Path(args.bundle), Path(args.asked), Path(args.answer)
kpack.check_pending(work)
asked: dict[str, Any] = kpack.read_json(asked_path)
if asked.get("schema") != ASKED_SCHEMA:
raise RunError(f"{asked_path} is not the guard's asked task ({ASKED_SCHEMA})")
asked_sha = krepair.file_sha256(asked_path)
refusals: list[krepair.Refusal] = []
for log in logs(work):
if json.loads(log.read_text(encoding="utf-8")).get("asked_sha256") == asked_sha:
refusals.append(krepair.Refusal("applied", f"{asked_path} (logged in {log})"))
if krepair.tree_sha256(bundle) != asked["bundle_sha256"]:
refusals.append(krepair.Refusal("stale-bundle", f"{bundle} is not the collection {asked_path} asked"))
if accepted_sha256(work) != asked.get("accepted_sha256"):
refusals.append(krepair.Refusal("stale-accepted", f"{kpack.k_work(work) / 'accepted.json'} changed since "
f"{asked_path} was asked; ask again"))
if refusals:
return krepair.refuse(refusals)
last = int(asked["pack"])
got = klink.read_guard(kpack.read_json(answer), asked["pages"], asked["sources"], asked["catalogue"],
klink.guard_candidates(bundle, asked["sources"]))
acc = kpack.read_accepted(work)
for src, pid in got.main_pages:
name = kpack.source_name(src)
acc["main_pages"] = [m for m in acc["main_pages"] if m["source"] != name] + [
{"source": name, "concept_id": pid, "pack": last, "by": "guard"}]
firsts = kpack.first_files(acc)
for pid, hid in got.homes:
if pid not in firsts:
raise RunError(f"the guard homed {pid}, which has no accepted fragment")
acc["homes"].append({"source": firsts[pid], "concept_id": pid, "home": hid, "pack": last, "by": "guard"})
dropped = kpack.drop_dangling(acc)
answered = {"homes": len(got.homes), "main_pages": len(got.main_pages), "none": got.none,
"unanswered": got.unanswered, "rejected": dict(sorted(got.rejected.items())),
**kpack.nonzero(dropped=dropped[0] + dropped[1])}
n = len(logs(work)) + 1
folder = work / "K-guard" / f"{LOG_PREFIX}{n}"
if folder.exists():
raise RunError(f"{folder} exists; a guard log is never written over")
entry = {"schema": LOG_SCHEMA, "at": krepair.now(), "asked": str(asked_path.resolve()), "asked_sha256": asked_sha,
"answer": str(answer.resolve()), "answer_sha256": krepair.file_sha256(answer),
"bundle_sha256": asked["bundle_sha256"], "pack": last, "answered": answered}
kpack.write_state(work, f"guard-{LOG_PREFIX}{n}", {"accepted.json": acc},
{folder / "guard.json": kpack.dump(entry)})
print(f"guard answer: homes {len(got.homes)}, main pages {len(got.main_pages)}, none {len(got.none)}, "
f"unanswered {len(got.unanswered)}, rejected {sum(got.rejected.values())} -> {folder / 'guard.json'}")
kpack.print_dropped(*dropped)
return 0
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="python3 -m okfwiki.kguard", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
a = sub.add_parser("ask")
for name in ("--work", "--bundle", "--sources", "--out"):
a.add_argument(name, required=True)
y = sub.add_parser("apply")
for name in ("--work", "--asked", "--answer", "--bundle"):
y.add_argument(name, required=True)
return p
COMMANDS = {"ask": cmd_ask, "apply": cmd_apply}
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
return COMMANDS[args.command](args)
except (RunError, OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
print(f"kguard cannot run: {exc}", file=sys.stderr)
return 2
except Exception: # an input this code did not foresee: still "could not run", with the traceback
traceback.print_exc()
return 2
if __name__ == "__main__":
sys.exit(main())

957
okfwiki/klink.py Normal file
View file

@ -0,0 +1,957 @@
"""The link pass: the model proposes links by meaning; this code strips, asks, accepts and places them. No model call.
Usage:
python3 -m okfwiki.klink strip BUNDLE --out DIR
python3 -m okfwiki.klink sentences --accepted FILE --pack N --bundle DIR --out FILE
python3 -m okfwiki.klink task BUNDLE --sentences FILE --catalogue IDS --out DIR
python3 -m okfwiki.klink accept PROPOSALS --bundle DIR --links FILE
python3 -m okfwiki.klink place BUNDLE --links FILE --out DIR [--replace]
python3 -m okfwiki.klink pass-before --accepted FILE --pack N [--to M] --bundle DIR [--catalogue IDS] --out DIR
python3 -m okfwiki.klink pass-after --pass-dir DIR --bundle DIR --links FILE --linked DIR --sources DIR
--transcripts DIR --okter FILE --work DIR --kost FILE --lines L[,L] --cap N --unit U --done D --total T
[--blocks FILE] [--name NAME]
Links are data (`links.json`, schema okf-wiki/links/1), never edits to pages, because K re-assembles pages from
fragments. A record is {page, sentence, anchor, at, target}: the sentence key, the anchor text and its character
offset in that sentence, and the target page. The sentence key is the first 16 hex characters of the sha256 of the
sentence's original text (sliced with audit.sentence_spans from the unmasked text) after footnote marks are removed,
every markdown link is reduced to its anchor text and whitespace is collapsed, plus "-<n>", the occurrence of that
text on the page. Every command first reduces each bundle link in a concept body to its anchor text, in prose, code,
headings and footnote definitions alike, so keys and offsets are computed on link-free text and placement is
deterministic. index.md and log.md are copied unchanged.
strip copies the bundle with every bundle link in a concept body reduced to its anchor text.
sentences the sentence keys on assembled pages that come from pack N's accepted fragments (`pack`, `concept_id`,
`text`). Exit 1 when the pack has no sentence (an empty denominator).
task renders the task parts: the contract, the catalogue as `id | title` lines and, per page, only the listed
sentences with their keys; parts are capped at 60,000 characters. A catalogue that alone exceeds the cap
cannot run (exit 2): it has to be split or filtered first.
accept validates proposals {"links": [{page, sentence, anchor, target}], "none": [ids]} and appends accepted
records; rejects are counted per reason, `none` answers are counted, and no record is ever rewritten.
place writes every record into a copy of the bundle; refuses an existing output unless --replace (then it writes
a temporary directory and swaps it in). A record whose sentence no longer exists is stale: exit 1.
pass-before / pass-after chain the non-model steps of one pass (sentences -> task; accept -> place -> link rule ->
cost harvest -> cost stop) and print only counts, so a pass costs the orchestrator few turns.
With --to M, pass-before covers packs N..M, one directory p<n> per pack, and pass-after on the parent
accepts every p<n> in pack order: the same links as one pass per pack, for the same orchestrator turns.
With --unit blocks, --done is the blocks of the accepted packs the pass covers (the done line of run-after
for its last pack), and the stop line divides by the blocks of every pack with a compiler run: in the
skill's chain the compiler of the next pack has run before this pass-after, so the harvest holds its run
and both sides of the stop line cover the same packs. The orchestration is one line for the whole session
and is not split per pack. A --done that is not those blocks, or a run on a capped line that is tied to no
pack or to a pack the log lacks, cannot run (exit 2, STOP). The stop is measured cost plus the blocks of a
compiled pack not yet accepted (cost stop --next, and --next-chars when the pack log has their characters), and the projection is reported only (the operator
12.10.2026).
The catalogue of a pack is filtered to its candidate targets within CATALOGUE_CHARS (filter_catalogue)
from a pool: the ids in --catalogue, or else the concepts with fragments in this or an earlier pack.
Exit 0 done, 1 a counted failure (stale, empty, STOP), 2 cannot run.
The guard (contract K2 point 17; kpack run-after calls it, it has no command of its own): `homeless` takes the pages
with a no-main-page-link finding and the sources with a main-page-missing finding from the auditor's JSON, so the
auditor is the only judge. `guard_parts` renders one task for the model with GUARD_CONTRACT, the pack's filtered
catalogue of main pages, each page without a home (id, title, description) and each source without a main page with
the pages that cite it; `read_guard` checks the answer: a home or none per page, a main page per source. The guard's
task runs on the linker agent, because a home is a link from a main page chosen by meaning.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import posixpath
import re
import shutil
import sys
import tempfile
from collections import Counter
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Any
from okfwiki import audit, cost
from okfwiki.common import RunError, write_new
SCHEMA = "okf-wiki/links/1"
SENTENCES_SCHEMA = "okf-wiki/link-sentences/1"
PARTS_SCHEMA = "okf-wiki/link-parts/1"
PART_CHARS = 60_000
CATALOGUE_CHARS = 32_000 # leaves ~27,000 characters of a part for sentences; order 28 A3: with the main pages first, the
# fewest far concepts a pack of the rest saw in the model was 205 (24,000: three packs under G's 179)
RECENT_PACKS = 2
MAX_LINKS = 3
PART_HEADER_ROOM = len("<!-- part 999 of 999 -->\n")
FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})")
REASONS = ("malformed", "sentence-missing", "target-missing", "self-link", "no-footnote", "anchor-not-in-sentence",
"anchor-in-code", "duplicate", "over-limit", "overlap")
CONTRACT = """## Contract
You add links to sentences of a knowledge collection. Every sentence below has a key. Link by meaning, not by name:
a link says that a phrase in the sentence is about a concept in the catalogue, and the phrase may use other words than
the concept's title.
- Targets: only ids from the catalogue in this task. Never link a page to itself.
- Anchor: a phrase that already stands, character for character, in the sentence, outside backticks. Change no word.
- At most 3 links in one sentence. No lists of related pages and no "see also" lines.
- When nothing in the catalogue fits, put the page id in `none`.
- Output: one JSON object `{"links": [{"page": ..., "sentence": ..., "anchor": ..., "target": ...}], "none": [...]}`,
where `page` is the page file as the task names it, `sentence` is the sentence key and `target` is a catalogue id.
Write it to the output path named in your prompt, then stop.
"""
GUARD_CONTRACT = """## Contract (home task)
A main page is the page for the main topic of one docs source. Every other page needs a home: a main page that links
to it in its list of related concepts. The pages and sources below lack one. Decide by meaning, as a reader of the
main page would look for the page.
- For each page under "Pages without a home": choose its home from the catalogue, or a page you name a main page
below, and add {"page": <id>, "home": <id>} to `homes`. When no main page fits, put the page id in `none`: a page
without a natural home is reported, never given a forced home.
- For each source under "Sources without a main page": choose, from the pages listed under it, the page whose topic is
that source's main topic, and add {"source": <path>, "page": <id>} to `main_pages`.
- Output: one JSON object `{"homes": [...], "none": [...], "main_pages": [...]}`. Write it to the output path named
in your prompt, then stop.
"""
GUARDED = frozenset({"no-main-page-link", "main-page-missing"})
GUARD_REASONS = ("malformed", "page-not-asked", "duplicate", "self-home", "home-not-in-catalogue", "source-not-asked",
"page-not-a-candidate")
KILDE_PATH = re.compile(r"(?<![\w.-])/?(kilde/[a-z0-9._-]*[a-z0-9_-])")
@dataclass(frozen=True)
class Sentence:
key: str
text: str # the original slice of the link-reduced body, footnote marks kept
masked: str # the same slice with inline code masked
start: int # offset of the slice in the link-reduced body
footnoted: bool
labels: tuple[str, ...]
# --- pages -----------------------------------------------------------------------------------------------------
def split_page(text: str) -> tuple[str, str]:
"""(frontmatter block with its closing fence and newline, body); a page without frontmatter has an empty head."""
lines = text.split("\n")
if not lines or lines[0].strip() != "---":
return "", text
end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
if end is None:
return "", text
return "\n".join(lines[:end + 1]) + "\n", "\n".join(lines[end + 1:])
def reduce_links(body: str) -> tuple[str, int]:
"""Every bundle link reduced to its anchor text, wherever it stands in the body; external links are kept."""
count = 0
def one(m: re.Match[str]) -> str:
nonlocal count
if not audit.is_bundle_link(m.group(2)):
return m.group(0)
count += 1
return m.group(1)
return audit.sub_links(one, body), count
def is_concept(rel: str) -> bool:
return rel.endswith(".md") and PurePosixPath(rel).name not in audit.RESERVED
def key_text(text: str) -> str:
plain = audit.sub_links(lambda m: m.group(1), audit.FOOTNOTE_REF.sub("", text))
return " ".join(plain.split())
def prose_spans(body: str) -> list[tuple[int, int]]:
"""(start, end) offsets of paragraphs and list items; headings, footnote definitions and fenced blocks are not prose."""
spans: list[tuple[int, int]] = []
current: list[int] | None = None
fence = ""
offset = 0
for raw in body.split("\n"):
start, end = offset, offset + len(raw)
offset = end + 1
f = FENCE.match(raw)
if fence:
if f and f.group(1)[0] == fence[0] and len(f.group(1)) >= len(fence):
fence = ""
continue
if f:
fence, current = f.group(1), None
continue
if not raw.strip() or audit.HEADING.match(raw) or audit.FOOTNOTE_DEF.match(raw.strip()):
current = None
continue
item = audit.LIST_ITEM.match(raw)
if item:
current = [start + item.end(), end]
spans.append((current[0], current[1]))
elif current is None:
current = [start + len(raw) - len(raw.lstrip()), end]
spans.append((current[0], current[1]))
else:
current[1] = end
spans[-1] = (current[0], current[1])
return spans
def body_sentences(body: str) -> list[Sentence]:
"""The sentences of a link-reduced body, each with its key."""
out: list[Sentence] = []
seen: Counter[str] = Counter()
for a, b in prose_spans(body):
block = body[a:b]
masked = audit.mask_code(block)
for s, e in audit.sentence_spans(block):
text = block[s:e]
norm = key_text(text)
digest = hashlib.sha256(norm.encode("utf-8")).hexdigest()[:16]
labels = tuple(audit.FOOTNOTE_REF.findall(masked[s:e]))
out.append(Sentence(f"{digest}-{seen[digest]}", text, masked[s:e], a + s, bool(labels), labels))
seen[digest] += 1
return out
def page_sentences(text: str) -> list[Sentence]:
return body_sentences(reduce_links(split_page(text)[1])[0])
@dataclass
class Page:
rel: str
head: str
body: str # link-reduced
title: str
resources: dict[str, str] # sources[].id -> resource
description: str = ""
def read_pages(bundle: Path) -> dict[str, Page]:
pages: dict[str, Page] = {}
for p in sorted(bundle.rglob("*.md")):
rel = p.relative_to(bundle).as_posix()
if not is_concept(rel):
continue
doc = audit.read_document(p, rel)
head, body = split_page(p.read_text(encoding="utf-8"))
fm = doc.frontmatter or {}
srcs = fm.get("sources")
resources = {str(s["id"]): str(s.get("resource", "")) for s in (srcs if isinstance(srcs, list) else [])
if isinstance(s, dict) and "id" in s}
pages[rel] = Page(rel, head, reduce_links(body)[0], str(fm.get("title") or PurePosixPath(rel).stem), resources,
" ".join(str(fm.get("description") or "").split()))
return pages
def page_rel(name: str) -> str:
name = name.strip()
return name if name.endswith(".md") else name + ".md"
def load_json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8"))
def dump(doc: Any) -> str:
return json.dumps(doc, ensure_ascii=False, indent=1) + "\n"
# --- strip and place -------------------------------------------------------------------------------------------
def copy_reduced(bundle: Path, out: Path, records: list[dict[str, Any]] | None) -> tuple[int, int, int]:
"""Copy bundle to out with every bundle link reduced, then place records; (links reduced, placed, stale)."""
by_page: dict[str, list[dict[str, Any]]] = {}
for r in records or []:
by_page.setdefault(str(r["page"]), []).append(r)
reduced = placed = stale = 0
for p in sorted(bundle.rglob("*")):
if not p.is_file():
continue
rel = p.relative_to(bundle).as_posix()
dst = out / rel
dst.parent.mkdir(parents=True, exist_ok=True)
if not is_concept(rel):
shutil.copyfile(p, dst)
continue
head, body = split_page(p.read_text(encoding="utf-8"))
body, n = reduce_links(body)
reduced += n
body, ok, bad = place_records(rel, body, by_page.pop(rel, []))
placed, stale = placed + ok, stale + bad
dst.write_text(head + body, encoding="utf-8")
stale += sum(len(v) for v in by_page.values()) # records for pages that no longer exist
return reduced, placed, stale
def place_records(rel: str, body: str, records: list[dict[str, Any]]) -> tuple[str, int, int]:
by_key = {s.key: s for s in body_sentences(body)}
edits: list[tuple[int, int, str]] = []
stale = 0
for r in records:
s = by_key.get(str(r["sentence"]))
anchor, at = str(r["anchor"]), int(r["at"])
if s is None or s.text[at:at + len(anchor)] != anchor:
stale += 1
continue
target = posixpath.relpath(str(r["target"]), posixpath.dirname(rel) or ".")
edits.append((s.start + at, s.start + at + len(anchor), f"[{anchor}]({target})"))
for a, b, link in sorted(edits, reverse=True):
body = body[:a] + link + body[b:]
return body, len(edits), stale
def strip(bundle: Path, out: Path) -> int:
if out.exists():
raise RunError(f"{out} exists; refusing to overwrite")
return copy_reduced(bundle, out, None)[0]
def place(bundle: Path, links: Path, out: Path, replace: bool) -> tuple[int, int]:
records = load_json(links)["links"]
if out.exists() and not replace:
raise RunError(f"{out} exists; refusing to overwrite without --replace")
out.parent.mkdir(parents=True, exist_ok=True)
tmp = Path(tempfile.mkdtemp(prefix=f".{out.name}-", dir=out.parent))
try:
_, placed, stale = copy_reduced(bundle, tmp, records)
if out.exists():
old = out.with_name(f".{out.name}-old")
out.rename(old)
tmp.rename(out)
shutil.rmtree(old)
else:
tmp.rename(out)
finally:
if tmp.exists():
shutil.rmtree(tmp)
return placed, stale
# --- what the model is asked -----------------------------------------------------------------------------------
def pack_sentences(accepted: dict[str, Any], pack: int, pages: dict[str, Page]) -> dict[str, list[str]]:
texts: dict[str, list[str]] = {}
for f in accepted["fragments"]:
if int(f["pack"]) == pack:
texts.setdefault(page_rel(str(f["concept_id"])), []).append(key_text(str(f["text"])))
out: dict[str, list[str]] = {}
for rel, frags in texts.items():
if rel not in pages:
continue
keys = [s.key for s in body_sentences(pages[rel].body)
if s.footnoted and any(key_text(s.text).rstrip(".!?") in t for t in frags)]
if keys:
out[rel] = keys
return out
def concept_id(rel: str) -> str:
return PurePosixPath(rel).with_suffix("").as_posix()
def pack_pool(accepted: dict[str, Any], pack: int, pages: dict[str, Page]) -> list[str]:
"""The concepts with fragments in this or an earlier pack: the pages that exist after pack N in an additive build."""
ids = {concept_id(page_rel(str(f["concept_id"]))) for f in accepted["fragments"] if int(f["pack"]) <= pack}
return sorted(cid for cid in ids if page_rel(cid) in pages)
def filter_catalogue(accepted: dict[str, Any], pack: int, pool: list[str], pages: dict[str, Page],
budget: int = CATALOGUE_CHARS) -> list[str]:
"""The candidate targets of a pack, ranked, cut at the first that does not fit the character budget.
The main pages (accepted.json's main_pages) come first, then the rest, each in rank order (order 28 A3: they are
the homes of criterion 6, and with the old rule a late pack of collection 1 saw few of them). Rank: (0) the
concepts this pack writes to; (1) those last written to in the RECENT_PACKS packs before it, latest first, because
the fixed order puts related sources next to each other; (2) the rest by how many distinct source files their page
cites, most first, so a late pack can still reach the central concepts made early. Ties by id. The rule uses
structure only (packs, citations and main pages), never names: the model links by meaning.
"""
last: dict[str, int] = {}
for f in accepted["fragments"]:
p, cid = int(f["pack"]), concept_id(page_rel(str(f["concept_id"])))
if p <= pack:
last[cid] = max(last.get(cid, 0), p)
def rank(cid: str) -> tuple[int, int, str]:
lp = last.get(cid)
if lp == pack:
return 0, 0, cid
if lp is not None and pack - lp <= RECENT_PACKS:
return 1, -lp, cid
return 2, -len(set(pages[page_rel(cid)].resources.values()) - {""}), cid
mains = {concept_id(page_rel(str(m["concept_id"]))) for m in accepted.get("main_pages", [])}
ranked = sorted(pool, key=rank)
out: list[str] = []
used = 0
for cid in [c for c in ranked if c in mains] + [c for c in ranked if c not in mains]:
size = len(catalogue_text([cid], pages))
if used + size > budget:
break
out.append(cid)
used += size
return out
@dataclass
class Section:
page: str
lines: list[str]
heading: str
def catalogue_text(ids: list[str], pages: dict[str, Page]) -> str:
out = []
for cid in ids:
rel = page_rel(cid)
if rel not in pages:
raise RunError(f"catalogue id {cid!r} is not a concept page in the bundle")
out.append(f"{PurePosixPath(rel).with_suffix('').as_posix()} | {pages[rel].title}\n")
return "".join(out)
def part_text(label: str, catalogue: str, sections: list[Section]) -> str:
body = "".join(f"### {s.heading}\n" + "".join(s.lines) + "\n" for s in sections)
return (f"# Link pass: {label}\n\n" + CONTRACT + "\n## Catalogue (link targets)\n\n" + catalogue
+ "\n## Sentences\n\n" + body)
def render_sections(label: str, fixed_ids: list[str], sections: list[Section], pages: dict[str, Page],
part_chars: int) -> list[tuple[str, dict[str, Any]]]:
fixed = catalogue_text(fixed_ids, pages)
if len(fixed) > part_chars:
raise RunError(f"the catalogue alone is {len(fixed)} characters ({len(fixed_ids)} concepts), over the part "
f"limit of {part_chars}: split or filter the catalogue before this pass")
room = part_chars - PART_HEADER_ROOM
def size(group: list[Section]) -> int:
return len(part_text(label, fixed, group))
if size([]) > room:
raise RunError(f"the contract and catalogue leave no room for a sentence within {part_chars} characters")
chunks: list[Section] = []
for s in sections:
piece = Section(s.page, [], s.heading)
for line in s.lines:
if piece.lines and size([Section(s.page, piece.lines + [line], piece.heading)]) > room:
chunks.append(piece)
piece = Section(s.page, [], s.heading + " (continued)")
piece.lines.append(line)
if size([piece]) > room:
raise RunError(f"one sentence of {s.page} does not fit in a part of {part_chars} characters")
chunks.append(piece)
groups: list[list[Section]] = []
for c in chunks:
if groups and size(groups[-1] + [c]) <= room:
groups[-1].append(c)
else:
groups.append([c])
out: list[tuple[str, dict[str, Any]]] = []
for i, group in enumerate(groups, 1):
text = f"<!-- part {i} of {len(groups)} -->\n" + part_text(label, fixed, group)
out.append((text, {"part": i, "file": f"part-{i}.md", "pages": len({s.page for s in group}),
"sentences": sum(len(s.lines) for s in group), "catalogue_chars": len(fixed),
"chars": len(text)}))
return out
def sentence_line(s: Sentence) -> str:
return f"- {s.key} | {' '.join(s.text.split())}\n"
def render_parts(bundle: Path, wanted: dict[str, list[str]], catalogue: list[str], label: str,
part_chars: int = PART_CHARS) -> list[tuple[str, dict[str, Any]]]:
return render_pages(read_pages(bundle), wanted, catalogue, label, part_chars)
def render_pages(pages: dict[str, Page], wanted: dict[str, list[str]], catalogue: list[str], label: str,
part_chars: int = PART_CHARS) -> list[tuple[str, dict[str, Any]]]:
sections = []
for rel, keys in wanted.items():
if rel not in pages:
raise RunError(f"{rel} is not a concept page in the bundle")
by_key = {s.key: s for s in body_sentences(pages[rel].body)}
missing = [k for k in keys if k not in by_key]
if missing:
raise RunError(f"{rel}: sentence keys not on the page: {', '.join(missing[:3])}")
sections.append(Section(rel, [sentence_line(by_key[k]) for k in keys], rel))
return render_sections(label, catalogue, sections, pages, part_chars)
def write_parts(out: Path, parts: list[tuple[str, dict[str, Any]]], extra: dict[str, Any] | None = None) -> None:
paths = [out / str(stats["file"]) for _, stats in parts] + [out / "parts.json"]
if any(p.exists() for p in paths):
raise RunError(f"task parts exist in {out}; refusing to overwrite")
for (text, _), p in zip(parts, paths):
write_new(p, text)
write_new(out / "parts.json", dump({"schema": PARTS_SCHEMA, **(extra or {}), "parts": [s for _, s in parts]}))
def read_ids(path: Path) -> list[str]:
ids: list[str] = []
for line in path.read_text(encoding="utf-8").splitlines():
cid = line.strip().removesuffix(".md")
if cid and cid not in ids:
ids.append(cid)
return ids
# --- the guard -------------------------------------------------------------------------------------------------
def kilde_path(text: str) -> str | None:
"""The first K1 path (kilde/...) in text, with or without a leading /."""
m = KILDE_PATH.search(text)
return m.group(1) if m else None
def homeless(audit_doc: Any) -> tuple[list[str], list[str]]:
"""(pages with no-main-page-link, K1 paths with main-page-missing) from the auditor's JSON, each once, sorted.
The auditor is the only judge (K2 point 17): nothing here counts links. A main-page-missing finding names its
source in `file` or, failing that, in `message`; one that names no kilde/ path cannot run."""
findings = audit_doc.get("findings") if isinstance(audit_doc, dict) else None
if not isinstance(findings, list) or not all(isinstance(f, dict) for f in findings):
raise RunError("the auditor's output has no list of findings")
pages: set[str] = set()
missing: set[str] = set()
for f in findings:
if f.get("check") == "no-main-page-link":
pages.add(page_rel(str(f.get("file") or "")))
elif f.get("check") == "main-page-missing":
path = kilde_path(str(f.get("file") or "")) or kilde_path(str(f.get("message") or ""))
if path is None:
raise RunError(f"a main-page-missing finding names no kilde/ source: {f!r}")
missing.add(path)
return sorted(pages), sorted(missing)
def guard_candidates(bundle: Path, sources: list[str]) -> dict[str, list[str]]:
"""Per K1 path, the ids of the pages that cite it: the only pages that can be its main page (K2 point 1)."""
pages = read_pages(bundle)
return {src: sorted(concept_id(rel) for rel, p in pages.items() if "/" + src in p.resources.values())
for src in sources}
def guard_line(cid: str, pages: dict[str, Page], described: bool) -> str:
rel = page_rel(cid)
if rel not in pages:
raise RunError(f"{rel} is not a concept page in the bundle")
p = pages[rel]
return f"- {cid} | {' '.join(p.title.split())}" + (f" | {p.description}" if described and p.description else "") + "\n"
def guard_parts(bundle: Path, homeless_pages: list[str], sources: list[str], catalogue: list[str], label: str,
part_chars: int = PART_CHARS) -> list[str]:
"""The guard's task: the contract and the catalogue of main pages in every part, then the pages without a home
and the sources without a main page (each with the pages that cite it), in parts of at most part_chars."""
pages = read_pages(bundle)
candidates = guard_candidates(bundle, sources)
head = (f"# Home task: {label}\n\n" + GUARD_CONTRACT + "\n## Catalogue (main pages)\n\n"
+ "".join(guard_line(cid, pages, False) for cid in catalogue) + "\n")
items: list[tuple[str, str]] = [("pages", guard_line(concept_id(page_rel(rel)), pages, True))
for rel in homeless_pages]
for src in sources:
cands = "".join(guard_line(cid, pages, True) for cid in candidates[src]) or "(no page cites it)\n"
items.append(("sources", f"### {src}\n\n{cands}\n"))
titles = {"pages": "## Pages without a home\n\n", "sources": "## Sources without a main page\n\n"}
def text(group: list[tuple[str, str]]) -> str:
out, kind = head, ""
for k, piece in group:
if k != kind:
out, kind = out + ("\n" if kind else "") + titles[k], k
out += piece
return out
room = part_chars - PART_HEADER_ROOM
if len(text([])) > room:
raise RunError(f"the guard's contract and catalogue leave no room for a page within {part_chars} characters")
groups: list[list[tuple[str, str]]] = []
for item in items:
if groups and len(text(groups[-1] + [item])) <= room:
groups[-1].append(item)
elif len(text([item])) <= room:
groups.append([item])
else:
raise RunError(f"one item of the guard's task does not fit in a part of {part_chars} characters")
return [f"<!-- part {i} of {len(groups)} -->\n" + text(g) for i, g in enumerate(groups, 1)]
@dataclass
class GuardAnswer:
homes: list[tuple[str, str]] # (page id, home id)
main_pages: list[tuple[str, str]] # (K1 path, page id)
none: list[str]
unanswered: list[str]
rejected: Counter[str]
def read_guard(answer: Any, asked_pages: list[str], asked_sources: list[str], catalogue: list[str],
candidates: dict[str, list[str]]) -> GuardAnswer:
"""The guard's answer as code accepts it: per source a main page among the pages that cite it; per page a home
from the catalogue or from the main pages this answer names, or none; the first valid answer for each counts."""
doc = answer if isinstance(answer, dict) else {}
asked = [concept_id(page_rel(p)) for p in asked_pages]
rejected: Counter[str] = Counter()
def entries(key: str) -> list[Any]:
value = doc.get(key)
return value if isinstance(value, list) else []
main_pages: list[tuple[str, str]] = []
for m in entries("main_pages"):
if not (isinstance(m, dict) and isinstance(m.get("source"), str) and isinstance(m.get("page"), str)):
rejected["malformed"] += 1
continue
src, pid = kilde_path(m["source"]), concept_id(page_rel(m["page"]))
if src not in asked_sources:
rejected["source-not-asked"] += 1
elif any(s == src for s, _ in main_pages):
rejected["duplicate"] += 1
elif pid not in candidates.get(src, []):
rejected["page-not-a-candidate"] += 1
else:
main_pages.append((src, pid))
named = {pid for _, pid in main_pages}
targets = set(catalogue) | named
homes: list[tuple[str, str]] = []
answered: set[str] = set()
for h in entries("homes"):
if not (isinstance(h, dict) and isinstance(h.get("page"), str) and isinstance(h.get("home"), str)):
rejected["malformed"] += 1
continue
pid, hid = concept_id(page_rel(h["page"])), concept_id(page_rel(h["home"]))
if pid not in asked:
rejected["page-not-asked"] += 1
elif pid in answered:
rejected["duplicate"] += 1
elif hid == pid:
rejected["self-home"] += 1
elif hid not in targets:
rejected["home-not-in-catalogue"] += 1
else:
homes.append((pid, hid))
answered.add(pid)
none: list[str] = []
for x in entries("none"):
if not isinstance(x, str):
rejected["malformed"] += 1
continue
pid = concept_id(page_rel(x))
if pid not in asked:
rejected["page-not-asked"] += 1
elif pid in answered:
rejected["duplicate"] += 1
else:
none.append(pid)
answered.add(pid)
unanswered = [pid for pid in asked if pid not in answered and pid not in named]
return GuardAnswer(homes, main_pages, none, unanswered, rejected)
# --- accept ----------------------------------------------------------------------------------------------------
def find_anchor(s: Sentence, anchor: str) -> tuple[int | None, str | None]:
"""(offset of the first occurrence outside code, None) or (None, reason)."""
if not anchor or any(c in anchor for c in "[]\n"):
return None, "anchor-not-in-sentence"
in_code = False
start = s.text.find(anchor)
while start >= 0:
end = start + len(anchor)
whole = (start == 0 or not s.text[start - 1].isalnum()) and (end == len(s.text) or not s.text[end].isalnum())
if whole:
if s.masked[start:end] == anchor:
return start, None
in_code = True
start = s.text.find(anchor, start + 1)
return None, "anchor-in-code" if in_code else "anchor-not-in-sentence"
def accept(proposals: dict[str, Any], bundle: Path, links: Path, origin: str) -> dict[str, Any]:
pages = read_pages(bundle)
sentences = {rel: {s.key: s for s in body_sentences(p.body)} for rel, p in pages.items()}
doc: dict[str, Any] = load_json(links) if links.is_file() else {
"schema": SCHEMA, "links": [], "none": [], "none_count": 0, "accepted": 0, "rejected": {}}
records: list[dict[str, Any]] = doc["links"]
rejected: Counter[str] = Counter()
accepted = 0
raw = proposals.get("links") if isinstance(proposals, dict) else None
for p in raw if isinstance(raw, list) else []:
reason, record = judge(p, pages, sentences, records)
if reason:
rejected[reason] += 1
else:
assert record is not None
records.append(record)
accepted += 1
none = proposals.get("none") if isinstance(proposals, dict) else None
nones = [str(x) for x in none] if isinstance(none, list) else []
doc["none"] += [{"id": x, "from": origin} for x in nones]
doc["none_count"] = len(doc["none"])
doc["accepted"] = len(records)
total = Counter(doc["rejected"])
total.update(rejected)
doc["rejected"] = {r: total[r] for r in REASONS if total[r]}
links.parent.mkdir(parents=True, exist_ok=True)
links.write_text(dump(doc), encoding="utf-8")
return {"accepted": accepted, "rejected": dict(rejected), "none": len(nones)}
def judge(p: Any, pages: dict[str, Page], sentences: dict[str, dict[str, Sentence]],
records: list[dict[str, Any]]) -> tuple[str | None, dict[str, Any] | None]:
if not (isinstance(p, dict) and all(isinstance(p.get(k), str) for k in ("page", "sentence", "anchor", "target"))):
return "malformed", None
rel, target, anchor, key = page_rel(p["page"]), page_rel(p["target"]), p["anchor"], p["sentence"]
s = sentences.get(rel, {}).get(key)
if s is None:
return "sentence-missing", None
if target not in pages:
return "target-missing", None
if target == rel:
return "self-link", None
if not s.footnoted:
return "no-footnote", None
at, reason = find_anchor(s, anchor)
if at is None:
return reason, None
here = [r for r in records if r["page"] == rel and r["sentence"] == key]
if any(r["target"] == target for r in here):
return "duplicate", None
if len(here) >= MAX_LINKS:
return "over-limit", None
if any(at < int(r["at"]) + len(str(r["anchor"])) and int(r["at"]) < at + len(anchor) for r in here):
return "overlap", None
return None, {"page": rel, "sentence": key, "anchor": anchor, "at": at, "target": target}
# --- command line ----------------------------------------------------------------------------------------------
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
s = sub.add_parser("strip")
s.add_argument("bundle")
s.add_argument("--out", required=True)
se = sub.add_parser("sentences")
for name in ("--accepted", "--bundle", "--out"):
se.add_argument(name, required=True)
se.add_argument("--pack", type=int, required=True)
t = sub.add_parser("task")
t.add_argument("bundle")
for name in ("--sentences", "--catalogue", "--out"):
t.add_argument(name, required=True)
a = sub.add_parser("accept")
a.add_argument("proposals")
a.add_argument("--bundle", required=True)
a.add_argument("--links", required=True)
pl = sub.add_parser("place")
pl.add_argument("bundle")
pl.add_argument("--links", required=True)
pl.add_argument("--out", required=True)
pl.add_argument("--replace", action="store_true")
b = sub.add_parser("pass-before")
for name in ("--accepted", "--bundle", "--out"):
b.add_argument(name, required=True)
b.add_argument("--catalogue")
b.add_argument("--pack", type=int, required=True)
b.add_argument("--to", type=int)
f = sub.add_parser("pass-after")
for name in ("--pass-dir", "--bundle", "--links", "--linked", "--sources", "--transcripts", "--okter", "--work",
"--kost", "--lines", "--unit"):
f.add_argument(name, required=True)
for name in ("--cap", "--done", "--total"):
f.add_argument(name, type=int, required=True)
f.add_argument("--blocks")
f.add_argument("--name")
return p
def sentences_doc(accepted: Path, pack: int, bundle: Path) -> dict[str, Any]:
wanted = pack_sentences(load_json(accepted), pack, read_pages(bundle))
return {"schema": SENTENCES_SCHEMA, "pack": pack, "pages": wanted}
def pass_before(args: argparse.Namespace) -> int:
accepted, pages = load_json(Path(args.accepted)), read_pages(Path(args.bundle))
given = read_ids(Path(args.catalogue)) if args.catalogue else None
last = args.pack if args.to is None else args.to
if last < args.pack:
raise RunError(f"--to {last} is before --pack {args.pack}")
jobs = []
for pack in range(args.pack, last + 1): # render every pack before writing any, so a failure leaves nothing behind
out = Path(args.out) if args.to is None else Path(args.out) / f"p{pack}"
wanted = pack_sentences(accepted, pack, pages)
pool = given if given is not None else pack_pool(accepted, pack, pages)
ids = filter_catalogue(accepted, pack, pool, pages)
jobs.append((pack, out, wanted, pool, ids, render_pages(pages, wanted, ids, f"pack {pack}")))
empty = 0
for pack, out, wanted, pool, ids, parts in jobs:
write_new(out / "sentences.json", dump({"schema": SENTENCES_SCHEMA, "pack": pack, "pages": wanted}))
write_parts(out, parts, {"pack": pack, "catalogue": {"concepts": len(ids), "pool": len(pool)}})
n = sum(len(v) for v in wanted.values())
empty += not n
cat = parts[0][1]["catalogue_chars"] if parts else 0
print(f"pass-before: pack {pack}, pages {len(wanted)}, sentences {n}, parts {len(parts)}, catalogue "
f"{len(ids)} of {len(pool)} concepts, {cat} characters -> {out}")
for _, stats in parts:
print(f" {out / str(stats['file'])} -> {out / ('proposals-' + str(stats['part']) + '.json')}")
return 1 if empty else 0
def counts(rejected: dict[str, int]) -> str:
return ", ".join(f"{r} {n}" for r, n in sorted(rejected.items())) or "none"
def pack_dirs(pass_dir: Path) -> list[Path]:
"""The p<n> directories of an interval, in pack order."""
dirs = [d for d in pass_dir.iterdir() if d.is_dir() and re.fullmatch(r"p\d+", d.name)]
return sorted(dirs, key=lambda d: int(d.name[1:]))
def accept_dir(directory: Path, bundle: Path, links: Path) -> tuple[int, int, Counter[str]]:
files = sorted(directory.glob("proposals-*.json"))
if not files:
raise RunError(f"no proposals-*.json in {directory}")
totals: Counter[str] = Counter()
accepted = nones = 0
for path in files:
r = accept(load_json(path), bundle, links, str(path))
accepted, nones = accepted + int(r["accepted"]), nones + int(r["none"])
totals.update(r["rejected"])
return accepted, nones, totals
def stop_blocks(kost: dict[str, Any], packs_log: dict[str, Any], lines: list[str], last: int,
done: int) -> tuple[int, list[int], int, int | None]:
"""The blocks done after a pass whose last pack is `last`: the packs up to `last` and every pack with a compiler
run in the harvest, so the measured cost and the blocks cover the same packs (order 22 session B2: the next pack's
compiler counted over the pass's blocks alone). `done` must be the blocks of the accepted packs up to `last`.
Second: the packs with a compiler run in this ledger, over whose blocks the stop line takes the next pack's share
(order 27 A3: packs built in an earlier ledger cost nothing here). Third: the blocks of the compiled packs not yet
accepted, whose compiler may run once more before the next check. Fourth: their characters (`block_chars`),
None when one of them has none (a log written before blocks had characters; motor B17)."""
blocks = {int(p["pack"]): int(p["blocks"]) for p in packs_log["packs"]}
covered = sum(int(p["blocks"]) for p in packs_log["packs"] if "accepted" in p and int(p["pack"]) <= last)
if done != covered:
raise RunError(f"--done {done} is not the blocks of the accepted packs up to pack {last} ({covered})")
compiled: set[int] = set()
for r in kost["runs"]:
if r["line"] not in lines:
continue
if r.get("pack") is None:
raise RunError(f"the run {r['description']!r} is tied to no pack: STOP")
if int(r["pack"]) not in blocks:
raise RunError(f"the run {r['description']!r} is for pack {r['pack']}, which the pack log lacks: STOP")
if r["line"] == "build":
compiled.add(int(r["pack"]))
built = {n for n in blocks if n <= last} | compiled
waiting = [p for p in packs_log["packs"] if int(p["pack"]) in built and "accepted" not in p]
pending = sum(int(p["blocks"]) for p in waiting)
chars = (sum(int(p["block_chars"]) for p in waiting)
if all(isinstance(p.get("block_chars"), int) for p in waiting) else None)
return sum(blocks[n] for n in built), sorted(compiled), pending, chars
def pass_after(args: argparse.Namespace) -> int:
pass_dir, bundle, links = Path(args.pass_dir), Path(args.bundle), Path(args.links)
dirs = pack_dirs(pass_dir)
for d in dirs: # every pack has its proposals before any is accepted, so a missing one changes nothing
if not any(d.glob("proposals-*.json")):
raise RunError(f"no proposals-*.json in {d}")
totals: Counter[str] = Counter()
accepted = nones = 0
for d in dirs or [pass_dir]:
a, n, t = accept_dir(d, bundle, links)
if dirs:
print(f"accept {d.name}: accepted {a}, rejected {sum(t.values())} ({counts(dict(t))}), none {n}")
accepted, nones = accepted + a, nones + n
totals.update(t)
print(f"accept: accepted {accepted}, rejected {sum(totals.values())} ({counts(dict(totals))}), none {nones}")
placed, stale = place(bundle, links, Path(args.linked), replace=True)
print(f"place: {placed} links placed, {stale} stale")
if stale:
return 1
report = audit.link_rule_report(audit.load_bundle(Path(args.linked), Path(args.sources)))
print(f"link rule: {report['pass']} of {report['of']}, misplaced {report['misplaced']}, broken {report['broken']}, "
f"max out-degree {report['max_out_degree']}")
harvest = ["harvest", "--transcripts", args.transcripts, "--okter", args.okter, "--work", args.work,
"--out", args.kost]
code = cost.main(harvest)
print(f"cost harvest: exit {code}")
if code:
return code
done, following, chars = args.done, None, None
if args.unit == "blocks":
if not args.blocks:
raise RunError("--unit blocks needs --blocks (the pack log)")
last = max(int(d.name[1:]) for d in dirs) if dirs else int(load_json(pass_dir / "sentences.json")["pack"])
done, packs, following, chars = stop_blocks(load_json(Path(args.kost)), load_json(Path(args.blocks)),
cost.capped_lines(args.lines), last, args.done)
print(f"done: {done} of {args.total} blocks (packs {', '.join(map(str, packs))} with a compiler run)")
stop = ["stop", "--lines", args.lines, "--cap", str(args.cap), "--unit", args.unit, "--done", str(done),
"--total", str(args.total), "--kost", args.kost]
stop += ["--blocks", args.blocks] if args.blocks else []
stop += ["--name", args.name] if args.name else []
stop += ["--next", str(following)] if following is not None else []
stop += ["--next-chars", str(chars)] if following is not None and chars is not None else []
code = cost.main(stop)
print(f"cost stop: exit {code}")
return code
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
if args.command == "strip":
n = strip(Path(args.bundle), Path(args.out))
print(f"strip: {n} links reduced to their anchor text -> {args.out}")
return 0
if args.command == "sentences":
doc = sentences_doc(Path(args.accepted), args.pack, Path(args.bundle))
write_new(Path(args.out), dump(doc))
n = sum(len(v) for v in doc["pages"].values())
print(f"sentences: pack {args.pack}, {len(doc['pages'])} pages, {n} sentences -> {args.out}")
return 0 if n else 1
if args.command == "pass-before":
return pass_before(args)
if args.command == "task":
out = Path(args.out)
doc = load_json(Path(args.sentences))
label = Path(args.sentences).stem
parts = render_parts(Path(args.bundle), doc["pages"], read_ids(Path(args.catalogue)), label)
write_parts(out, parts)
n = sum(len(v) for v in doc["pages"].values())
cat = parts[0][1]["catalogue_chars"] if parts else 0
print(f"{args.command}: pages {len(doc['pages'])}, sentences {n}, parts {len(parts)}, catalogue {cat} "
f"characters -> {out}")
for _, stats in parts:
print(f" {out / str(stats['file'])} -> {out / ('proposals-' + str(stats['part']) + '.json')}")
return 0 if n else 1
if args.command == "accept":
r = accept(load_json(Path(args.proposals)), Path(args.bundle), Path(args.links), args.proposals)
print(f"accept: accepted {r['accepted']}, rejected {sum(r['rejected'].values())} "
f"({counts(r['rejected'])}), none {r['none']}")
return 0
if args.command == "place":
placed, stale = place(Path(args.bundle), Path(args.links), Path(args.out), args.replace)
print(f"place: {placed} links placed, {stale} stale -> {args.out}")
return 1 if stale else 0
return pass_after(args)
except (RunError, OSError, json.JSONDecodeError, KeyError, ValueError, TypeError) as exc:
print(f"klink cannot run: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())

1012
okfwiki/kmerge.py Normal file

File diff suppressed because it is too large Load diff

1992
okfwiki/kpack.py Normal file

File diff suppressed because it is too large Load diff

1212
okfwiki/krepair.py Normal file

File diff suppressed because it is too large Load diff

546
okfwiki/reader.py Normal file
View file

@ -0,0 +1,546 @@
"""The tool around the reader pass for PLAN.md criterion 2 ("slutning støttet"). It calls no model.
The reader judges up to ten claims per call and gives one verdict per claim (K3 in the wave-1 contracts). The model
call happens in the user's own Claude Code session: one isolated subagent per call, described "V1-<run>-read-<call>",
which reads one rendered task file and replies with one line per claim. The agent definition
(agents/okf-wiki-reader.md) is thin: it only tells the subagent to read its task file and follow it. The instructions
travel in the task (INSTRUCTIONS below), so a change between runs reaches the reader whatever the session loaded, and
the transcript shows word for word what the reader was told.
render writes one task per call of a call layout (okf-wiki/reader-calls/1, bound to the set by its sha256):
the instructions, then the claims numbered 1. to n. in layout order, each with its excerpts (file, line,
text). Nothing else reaches the reader: no id, no call name, no label, no other call. It refuses to write
when the agent's system text or the instructions share a run of eight words with any claim or excerpt of
the set, so the reader cannot carry text from the pairs it is judged on, and it never changes a task
already written.
collect reads every reader's reply from the subagent transcripts (<transcripts>/<session>/subagents/, the
or <session>/subagents/workflows/<id>/ for a workflow; the agent type okf-wiki-reader, bare or
plugin-namespaced as <plugin>:okf-wiki-reader, and the
description "V1-<run>-read-<call>"), the message it handed back or else its last text, and turns it
into the verdicts file that gates/calibration.py scores ({"reader": "<name>", "verdicts": [{"id",
"verdict"}]}). One rule, fail-closed (K3 point 3): every stripped, non-empty line must be
"<1-10>: supported" or "<1-10>: unsupported"; one line that does not fit, or a number outside 1..n,
gives no verdict for any claim of the call; a number given twice or not at all gives none for that
claim. A call answered by two transcripts gets no verdict.
It also names the models the readers ran on (PLAN § Rammer 23): the sorted, distinct `message.model`
of the assistant records in the transcripts it turns into verdicts (records without one, and
"<synthetic>", are left out), written as "models" beside "reader" and "verdicts" and printed as
"models: ...". One of two choices is required: --calibration says this run is a calibration gate run
and is not checked; --gate-passed FILE names the record of green gate runs (gates/calibration.py gate
--passed; --gate-passed alone reads the tracked gates/calibration.passed.json, which holds only models
and sha256, so a fresh clone without the set can still read), and when the transcripts name no model, or a model the record does not hold with a run on the
set and key sha256 of every calibration manifest (--manifest, once per manifest; default the tracked
gates/calibration.manifest.json, the Claude Code docs set, and gates/calibration-folder.manifest.json,
the folder set of Norwegian and English PDF/Office documents), collect writes nothing and exits 1, so a
reading on a model that never passed both gates gives no verdicts.
Exit 0 = written with every verdict; 1 = written with verdicts missing, refused, or could not run.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from okfwiki.transcript import is_agent_type, read_transcript
SUPPORTED = "supported"
UNSUPPORTED = "unsupported"
CALLS_SCHEMA = "okf-wiki/reader-calls/1"
CALL_SIZE_MAX = 10
RUN_WORDS = 8
WORD = re.compile(r"[a-z0-9]+(?:['’.-][a-z0-9]+)*")
LINE = re.compile(r"([1-9]|10)\s*:\s*(supported|unsupported)")
CALL_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*")
HANDBACK = "SubagentHandback"
AGENT_TYPE = "okf-wiki-reader"
SYNTHETIC_MODEL = "<synthetic>"
PASSED_SCHEMA = "okf-wiki/calibration-passed/1"
CALIBRATION_MANIFEST = Path(__file__).resolve().parents[1] / "gates" / "calibration.manifest.json"
CALIBRATION_FOLDER_MANIFEST = Path(__file__).resolve().parents[1] / "gates" / "calibration-folder.manifest.json"
CALIBRATION_PASSED = Path(__file__).resolve().parents[1] / "gates" / "calibration.passed.json" # models and sha256 only
INSTRUCTIONS = """\
# How to judge the claims
You are an independent reader. This task gives you a numbered list of claims from a knowledge collection. Under each
claim stand the source excerpts it cites. For every claim, decide whether its own excerpts support it, and give one
verdict per claim.
Judge each claim on its own. Only the excerpts printed under a claim count as evidence for that claim. The other claims
in this task and their excerpts are not evidence for it, and the verdict on one claim must never lean on the verdict on
another. Any number of the claims may be unsupported: none, one, several or all of them. Do not expect a pattern, and
do not let a run of supported claims lower your guard on the next one.
You have not seen the pages the claims come from, and you must not look for them. What you know about the subject from
anywhere else does not count, even when you are sure it is true: a claim that is true in the world but not stated by
its excerpts is unsupported.
## What "supported" means
A claim is **supported** when everything it says is stated by its excerpts, or follows from them in one step that no
careful reader of the excerpts could deny. Otherwise it is **unsupported**. There is no middle verdict: a claim that is
mostly right is unsupported.
Allowed, and still supported:
- Different words for the same thing: synonyms, reordering, active for passive, a plain-language summary.
- Saying less than the excerpt: leaving out details, examples, conditions that only narrow what the excerpt says
further, or items from a list, as long as what remains is still exactly true. A claim that is less specific than its
source is fine; a claim that is more general is not.
- One necessary step: a conclusion that the excerpt forces (if it says every X has Y and that Z is an X, then Z has Y;
if it gives two numbers, their plain sum or difference). If the step needs any assumption of your own, it is not
necessary.
- Combining excerpts: when the claim joins facts from two excerpts, each part is backed by the excerpt that states it,
and the join itself adds nothing new.
## How to read one claim
1. Break the claim into its parts: every subject, every thing said about it, every number and unit, every qualifier
(all, always, only, never, some, may, by default, at most, at least, when, unless), every stated reason or order of
events, and every negation.
2. For each part, find the words in this claim's own excerpts that state it. If you cannot point to them, that part is
not supported.
3. Then run the six checks below on the claim as a whole. Each one describes a way a claim fails even though it sounds
close to its source. One failed check makes the claim unsupported.
## The six checks
1. **Broader than the source.** The claim widens what the excerpt says: "some", "can", "may", "often", "by default" or
"in this case" becomes "all", "always", "must", "every" or "in general"; a condition or exception in the excerpt is
dropped so the claim covers cases the excerpt does not; one example is turned into a rule; one product, platform,
version or setting is turned into all of them.
2. **A number that differs.** Every number, count, limit, size, duration, date, version, percentage and unit in the
claim must be the one in the excerpts, or the result of one plain calculation on them. A different figure, a rounded
figure presented as exact, a swapped unit, or "at least" where the excerpt says "at most" fails. When the claim
gives a figure the excerpts do not print (a difference, sum, total, share, ratio, count of listed items, or a span
between two dates), work it out yourself from the excerpts' own figures before you judge; if your result is not
the claim's figure, the claim fails, however close it is and however natural the sentence reads.
3. **Cause, purpose or order that the source does not give.** The claim says A causes, enables, requires, prevents or
happens before B, or that A is done in order to get B. The excerpts must state that same link in the same direction.
Two facts that the excerpt only lists side by side are not a cause. A reversed direction (B causes A, B is needed
for A) fails.
4. **Something not in the excerpts.** Any subject, feature, option, value, reason, consequence, comparison or
recommendation in the claim that the excerpts do not mention fails, however plausible it is. Look hardest at the
last clause of the claim and at small added words ("also", "instead", "automatically", "only").
5. **Negation or direction reversed.** The claim says the opposite of the excerpt on any point: can versus cannot, is
versus is not, enabled versus disabled, allowed versus blocked, recommended versus discouraged, required versus
optional, increases versus decreases, included versus excluded.
6. **A property moved between different files.** When a claim's excerpts come from different files, they may be about
different subjects. Each property in the claim must belong to the subject that its own excerpt attaches it to. A
claim that gives one file's subject a property that only the other file states for its own subject fails, even
when every word appears somewhere in the excerpts.
## Deciding
- If every part is backed and all six checks pass, the verdict is supported. Do not reject a claim for being short,
for leaving things out, for style, or because you would have written it differently.
- If any part is not backed, or any check fails, the verdict is unsupported.
- When you are unsure whether a step is necessary or only likely, it is not necessary: unsupported.
- Give every claim the same full reading, the last one as much as the first.
## Your reply
Work through every claim before you reply. Your reply is the list of verdicts and nothing else: one line per claim,
`<number>: supported` or `<number>: unsupported`, where `<number>` is the claim's number in this task (1, 2, 3 and so
on, without a leading zero) and the verdict is written in lower case. Give every number exactly once. Write nothing
else in the reply: no heading, no explanation, no reasoning, no code block, no markup, and no text before or after the
lines. Any other text in the reply makes the whole reply count as unanswered.
For a task with three claims, the reply has this form:
1: supported
2: unsupported
3: supported
"""
class RunError(Exception):
"""The command cannot run; reported as FAIL with exit 1."""
@dataclass(frozen=True)
class Excerpt:
file: str
line: int
text: str
@dataclass(frozen=True)
class Pair:
id: str
claim: str
excerpts: tuple[Excerpt, ...]
@dataclass(frozen=True)
class Call:
name: str
ids: tuple[str, ...]
def instructions(agent_text: str) -> str:
"""The body of an agent definition, without its YAML frontmatter."""
if agent_text.startswith("---\n"):
end = agent_text.find("\n---\n", 4)
if end < 0:
raise RunError("the agent definition opens a frontmatter block that never closes")
agent_text = agent_text[end + len("\n---\n") :]
return agent_text.strip()
def _read_json(path: Path, what: str) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError) as exc:
raise RunError(f"cannot read the {what} {path}: {exc}") from exc
def load_pairs(path: Path) -> list[Pair]:
"""Claims with their excerpts from a set file ({"pairs": [{id, claim, excerpts: [{file, line, text}]}]})."""
data = _read_json(path, "set")
raw = data.get("pairs") if isinstance(data, dict) else None
if not isinstance(raw, list):
raise RunError(f"{path} has no list of pairs")
pairs: list[Pair] = []
seen: set[str] = set()
for p in raw:
pid, claim, excerpts = (p.get(k) if isinstance(p, dict) else None for k in ("id", "claim", "excerpts"))
if not isinstance(pid, str) or not isinstance(claim, str) or not isinstance(excerpts, list) or not excerpts:
raise RunError(f"pair {pid!r} needs a string id, a string claim and at least one excerpt")
if pid in seen:
raise RunError(f"pair id {pid} is given twice")
seen.add(pid)
parsed = []
for e in excerpts:
if not (isinstance(e, dict) and isinstance(e.get("file"), str) and isinstance(e.get("text"), str)
and isinstance(e.get("line"), int)):
raise RunError(f"pair {pid}: every excerpt needs a file, a line and a text")
parsed.append(Excerpt(e["file"], e["line"], e["text"]))
pairs.append(Pair(pid, claim, tuple(parsed)))
return pairs
def load_calls(path: Path, set_path: Path) -> list[Call]:
"""The calls of a layout (K3 point 1), bound to the set file by its sha256. A layout may hold part of the set
(a sample); every id it names must be in the set and stand in exactly one call of at most call_size_max claims."""
data = _read_json(path, "call layout")
if not isinstance(data, dict) or data.get("schema") != CALLS_SCHEMA:
raise RunError(f"{path} is not a call layout with schema {CALLS_SCHEMA}")
size_max = data.get("call_size_max")
if not isinstance(size_max, int) or isinstance(size_max, bool) or not 1 <= size_max <= CALL_SIZE_MAX:
raise RunError(f"call_size_max {size_max!r} is not a whole number from 1 to {CALL_SIZE_MAX}")
digest = hashlib.sha256(set_path.read_bytes()).hexdigest()
if data.get("set_sha256") != digest:
raise RunError(f"the layout is bound to set {data.get('set_sha256')!r}, not to {set_path} ({digest})")
raw = data.get("calls")
if not isinstance(raw, list) or not raw:
raise RunError(f"{path} has no calls")
known = {p.id for p in load_pairs(set_path)}
calls: list[Call] = []
names: set[str] = set()
placed: set[str] = set()
for c in raw:
name, ids = (c.get("call"), c.get("ids")) if isinstance(c, dict) else (None, None)
if not isinstance(name, str) or not CALL_NAME.fullmatch(name):
raise RunError(f"call name {name!r} is not letters, digits, '-' and '_'")
if name in names:
raise RunError(f"call name {name} is given twice")
names.add(name)
if not isinstance(ids, list) or not ids or not all(isinstance(i, str) for i in ids):
raise RunError(f"call {name} has no list of claim ids")
if len(ids) > size_max:
raise RunError(f"call {name} has {len(ids)} claims, more than {size_max}")
for i in ids:
if i not in known:
raise RunError(f"call {name} names claim {i}, which is not in the set")
if i in placed:
raise RunError(f"claim {i} stands in more than one place")
placed.add(i)
calls.append(Call(name, tuple(ids)))
return calls
def render_call(call: Call, pairs: dict[str, Pair]) -> str:
"""One task: the instructions, then the claims numbered 1. to n., each with its excerpts."""
n = len(call.ids)
parts = [INSTRUCTIONS.strip(), "", "---", "", "## The claims", "",
f"This task has {n} claim{'s' if n != 1 else ''}, numbered 1 to {n}."]
for number, cid in enumerate(call.ids, 1):
p = pairs[cid]
parts += ["", f"{number}. <claim>{p.claim}</claim>"]
for e in p.excerpts:
parts += [f'<excerpt file="{e.file}" line="{e.line}">', e.text, "</excerpt>"]
return "\n".join(parts) + "\n"
def _runs(text: str) -> set[tuple[str, ...]]:
words = WORD.findall(text.lower())
return {tuple(words[i : i + RUN_WORDS]) for i in range(len(words) - RUN_WORDS + 1)}
def shared_runs(text: str, pairs: list[Pair]) -> list[str]:
"""Pair ids whose claim or excerpts share a run of RUN_WORDS words with an instruction text."""
mine = _runs(text)
return [p.id for p in pairs if mine & _runs(" ".join([p.claim, *(e.text for e in p.excerpts)]))]
def parse_reply(reply: str, n: int) -> dict[int, str]:
"""K3 point 3: the verdict per claim number of one call of n claims; a claim with no verdict is left out."""
lines = [line.strip() for line in reply.splitlines() if line.strip()]
found: dict[int, list[str]] = {}
for line in lines:
m = LINE.fullmatch(line)
if m is None:
return {}
number = int(m.group(1))
if number > n:
return {}
found.setdefault(number, []).append(m.group(2))
return {k: v[0] for k, v in sorted(found.items()) if len(v) == 1}
def final_reply(path: Path) -> str:
"""The reader's reply in its transcript: the message of its last SubagentHandback call (how Claude Code 2.1.295
delivers a subagent's final report), or else the text of its last assistant message (its text blocks joined in
order). Thinking is never part of the reply."""
records = [r for r in read_transcript(path) if r.get("type") == "assistant" and isinstance(r.get("message"), dict)]
handed: list[Any] = []
for r in records:
content = r["message"].get("content")
for block in content if isinstance(content, list) else []:
if isinstance(block, dict) and block.get("type") == "tool_use" and block.get("name") == HANDBACK:
given = block.get("input")
handed.append(given.get("message") if isinstance(given, dict) else None)
if handed:
return handed[-1] if isinstance(handed[-1], str) else ""
if not records:
return ""
last = records[-1]["message"].get("id")
texts: list[str] = []
for r in records:
if r["message"].get("id") != last:
continue
content = r["message"].get("content")
for block in content if isinstance(content, list) else []:
if isinstance(block, dict) and block.get("type") == "text" and isinstance(block.get("text"), str):
texts.append(block["text"])
return "\n".join(texts)
def transcript_models(path: Path) -> set[str]:
"""The models named by the assistant records of one transcript (message.model, as Claude Code writes it); records
without one, and the "<synthetic>" records Claude Code writes itself, name none."""
found: set[str] = set()
for r in read_transcript(path):
message = r.get("message") if r.get("type") == "assistant" else None
model = message.get("model") if isinstance(message, dict) else None
if isinstance(model, str) and model and model != SYNTHETIC_MODEL:
found.add(model)
return found
PASSED_FIELDS = ("model", "set_sha256", "key_sha256")
def read_passed(path: Path) -> list[dict[str, Any]]:
"""The entries of a record of green calibration runs (okf-wiki/calibration-passed/1), the one parser both
gates/calibration.py gate --passed (which adds to it) and collect --gate-passed use. A record that cannot be read,
does not have that form, or has an entry without a model, set_sha256 or key_sha256 string, is a RunError."""
data = _read_json(path, "record of green calibration runs")
entries = data.get("passed") if isinstance(data, dict) and data.get("schema") == PASSED_SCHEMA else None
if not isinstance(entries, list):
raise RunError(f"{path} is not a record of green calibration runs with schema {PASSED_SCHEMA}")
for e in entries:
for name in PASSED_FIELDS:
value = e.get(name) if isinstance(e, dict) else None
if not isinstance(value, str) or not value:
raise RunError(f"{path}: an entry of the record of green calibration runs names no {name}")
return entries
def manifest_shas(path: Path) -> tuple[str, str]:
"""(set sha256, key sha256) of a calibration manifest (gates/calibration.manifest.json)."""
data = _read_json(path, "calibration manifest")
shas = [data.get(part, {}).get("sha256") if isinstance(data, dict) and isinstance(data.get(part), dict) else None
for part in ("set", "key")]
if not all(isinstance(s, str) and s for s in shas):
raise RunError(f"{path} is not a calibration manifest with a set and a key sha256")
return str(shas[0]), str(shas[1])
def passed_models(path: Path, manifest: Path = CALIBRATION_MANIFEST) -> set[str]:
"""The models with a green run on the set and key the calibration manifest names: an entry whose set_sha256 or
key_sha256 differs (a toy set, an older set, a set rebuilt later) clears no model."""
set_sha, key_sha = manifest_shas(manifest)
return {e["model"] for e in read_passed(path) if e["set_sha256"] == set_sha and e["key_sha256"] == key_sha}
def manifests_of(args: argparse.Namespace) -> list[Path]:
"""The calibration manifests a green run must be on: those given with --manifest, else both tracked sets (the
Claude Code docs set and the folder set of Norwegian and English PDF/Office documents, motor B16)."""
given: list[str] | None = args.manifest
return [Path(m) for m in given] if given else [CALIBRATION_MANIFEST, CALIBRATION_FOLDER_MANIFEST]
def reader_transcripts(transcripts: Path, run: str) -> dict[str, list[Path]]:
"""Transcripts per call name of the reader agents of one run, found by their description and agent type, whether
started from the session (<session>/subagents/) or from a workflow (<session>/subagents/workflows/<id>/)."""
prefix = re.compile(rf"^V1-{re.escape(run)}-read-(\S+)$")
found: dict[str, list[Path]] = {}
metas = [*transcripts.glob("*/subagents/agent-*.meta.json"),
*transcripts.glob("*/subagents/workflows/*/agent-*.meta.json")]
for meta_path in sorted(metas):
try:
meta = json.loads(meta_path.read_text(encoding="utf-8"))
except (OSError, ValueError):
continue
if not isinstance(meta, dict) or not is_agent_type(meta.get("agentType"), AGENT_TYPE):
continue
m = prefix.match(str(meta.get("description", "")))
if m is None:
continue
path = meta_path.with_name(meta_path.name.removesuffix(".meta.json") + ".jsonl")
found.setdefault(m.group(1), []).append(path)
return found
@dataclass
class Collected:
doc: dict[str, Any]
claims: int
answered: int = 0
broke_rule: int = 0
twice: int = 0
unanswered: int = 0
stray: int = 0
def collect(name: str, calls: list[Call], found: dict[str, list[Path]]) -> Collected:
verdicts: list[dict[str, str]] = []
models: set[str] = set()
out = Collected({"reader": name, "models": [], "verdicts": verdicts}, sum(len(c.ids) for c in calls))
names = {c.name for c in calls}
out.stray = sum(len(paths) for call, paths in found.items() if call not in names)
for call in calls:
paths = found.get(call.name, [])
if not paths:
out.unanswered += 1
continue
if len(paths) > 1:
out.twice += 1
continue
out.answered += 1
if paths[0].is_file():
models |= transcript_models(paths[0])
given = parse_reply(final_reply(paths[0]) if paths[0].is_file() else "", len(call.ids))
if not given:
out.broke_rule += 1
for number, cid in enumerate(call.ids, 1):
if number in given:
verdicts.append({"id": cid, "verdict": given[number]})
out.doc["models"] = sorted(models)
return out
def cmd_render(args: argparse.Namespace) -> int:
set_path = Path(args.set)
pairs = load_pairs(set_path)
calls = load_calls(Path(args.calls), set_path)
try:
system = instructions(Path(args.agent).read_text(encoding="utf-8"))
except OSError as exc:
raise RunError(f"cannot read the agent definition {args.agent}: {exc}") from exc
for what, text in (("the agent's system text", system), ("the instructions in the task", INSTRUCTIONS)):
leaks = shared_runs(text, pairs)
if leaks:
raise RunError(f"{what} and the set share a run of {RUN_WORDS} words in {len(leaks)} pair(s); "
"nothing written")
by_id = {p.id: p for p in pairs}
out = Path(args.out_dir)
tasks = {out / f"{c.name}.md": render_call(c, by_id) for c in calls}
changed = sorted(p.name for p, t in tasks.items() if p.exists() and p.read_text(encoding="utf-8") != t)
if changed:
raise RunError(f"{len(changed)} task(s) already written with other text, first {changed[0]}; nothing written")
out.mkdir(parents=True, exist_ok=True)
for path, task in tasks.items():
path.write_text(task, encoding="utf-8")
print(f"render: {len(tasks)} tasks for {sum(len(c.ids) for c in calls)} claims written to {out}")
return 0
def cmd_collect(args: argparse.Namespace) -> int:
set_path = Path(args.set)
calls = load_calls(Path(args.calls), set_path)
transcripts = Path(args.transcripts)
if not transcripts.is_dir():
raise RunError(f"the transcripts directory {transcripts} does not exist")
got = collect(args.reader, calls, reader_transcripts(transcripts, args.run))
models: list[str] = got.doc["models"]
if args.gate_passed is not None:
for manifest in manifests_of(args):
passed = passed_models(Path(args.gate_passed), manifest)
lacking = [m for m in models if m not in passed] if models else ["(none)"]
if lacking:
print(f"collect cannot run: model(s) {', '.join(lacking)} have no green calibration run in "
f"{args.gate_passed} on the set and key of {manifest}; run the calibration gate on them first",
file=sys.stderr)
return 1
Path(args.out).write_text(json.dumps(got.doc, indent=1) + "\n", encoding="utf-8")
written = len(got.doc["verdicts"])
stray = f"; {got.stray} transcript{'s' if got.stray != 1 else ''} for a call not in the layout" if got.stray else ""
print(f"collect {args.reader}: {written} of {got.claims} verdicts written to {args.out}")
print(f"calls: {got.answered} of {len(calls)} answered; {got.broke_rule} broke the reply rule; "
f"{got.twice} answered twice; {got.unanswered} not answered{stray}")
print(f"missing: {got.claims - written}")
print(f"models: {', '.join(models) if models else '(none)'}")
return 0 if written == got.claims else 1
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
r = sub.add_parser("render", help="write one reader task per call")
r.add_argument("--set", required=True)
r.add_argument("--calls", required=True, help="the call layout (okf-wiki/reader-calls/1)")
r.add_argument("--agent", required=True, help="the reader's agent definition (agents/okf-wiki-reader.md)")
r.add_argument("--out-dir", required=True)
c = sub.add_parser("collect", help="turn the readers' replies in the transcripts into a verdicts file")
c.add_argument("--set", required=True)
c.add_argument("--calls", required=True)
c.add_argument("--transcripts", required=True,
help="~/.claude/projects/<slug>/ of the session that ran the readers")
c.add_argument("--run", required=True, help="the run name in the descriptions V1-<run>-read-<call>")
c.add_argument("--reader", required=True)
c.add_argument("--out", required=True)
check = c.add_mutually_exclusive_group(required=True)
check.add_argument("--calibration", action="store_true",
help="this run is a calibration gate run; the readers' model is not checked")
check.add_argument("--gate-passed", metavar="FILE", default=None, nargs="?", const=str(CALIBRATION_PASSED),
help="the record of green calibration runs (gates/calibration.py gate --passed; without FILE "
"the tracked gates/calibration.passed.json); collect writes nothing unless every model "
"the readers ran on is in it")
c.add_argument("--manifest", action="append", default=None,
help="a calibration manifest whose set and key a green run must be on, once per manifest; every "
"one must hold (default: the tracked gates/calibration.manifest.json and "
"gates/calibration-folder.manifest.json)")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
commands = {"render": cmd_render, "collect": cmd_collect}
try:
return commands[args.command](args)
except (RunError, OSError) as exc:
print(f"FAIL: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())

429
okfwiki/readers.py Normal file
View file

@ -0,0 +1,429 @@
"""A reader per file type: every unit of text in a document with its exact place. No model call (CLAUDE.md rule 1).
Usage:
python3 -m okfwiki.readers locate FILE QUERIES
Places, as the file-type gate defines them (gates/filetypes.manifest.json): pdf {"page": N}, the physical page;
docx {"paragraph": N}, the w:p elements of the body in order, empty ones and those in tables included, those in a text
box or a compatibility fallback not; xlsx {"sheet": name, "cell": "B7"}; pptx {"slide": N, "notes": bool}, the slide
list's order, hidden slides included; md, txt and html {"line": N}, for html the raw line of a visible text node
(not in head, title, script, style, noscript, template or svg). Numbers are 1-based.
Hidden Office content (a hidden sheet, row or column, a hidden slide, hidden text, a tracked deletion) is kept apart
from what a reader of the document sees. `locate` marks it hidden; `render` leaves it out and counts it.
Office files are read from their package XML with the standard library. PDF needs pypdf at exactly PDF_PIN, because
the quote check needs the same raw text on every run (rule decisions R9 and R37); any other version is refused.
locate FILE and a JSON file {"file", "queries": [{"id", "quote"}]}: prints {"answers": [{"id", "found":
[{"locator", "hidden"}]}]}, every place where the quote occurs, runs of whitespace collapsed to one space.
Exit 0 = done, 2 = could not run.
"""
from __future__ import annotations
import argparse
import html.parser
import json
import logging
import posixpath
import re
import sys
import zipfile
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from xml.etree import ElementTree
PDF_PIN = "6.14.2"
OFFICE = (".docx", ".xlsx", ".pptx")
TYPES = (".pdf",) + OFFICE
W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
S = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
P = "{http://schemas.openxmlformats.org/presentationml/2006/main}"
A = "{http://schemas.openxmlformats.org/drawingml/2006/main}"
R = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}"
PKG = "{http://schemas.openxmlformats.org/package/2006/relationships}"
MC_FALLBACK = "{http://schemas.openxmlformats.org/markup-compatibility/2006}Fallback"
SPACE = re.compile(r"\s+")
# A line of a rendering that Markdown would read as structure gets a backslash, so a paragraph is always a paragraph.
STRUCTURE = re.compile(r"^(#|```|~~~|[-*+] |\d+[.)] |>|\||<)")
# A sheet name heads its rows as their section, below the levels the skeleton proposes as concepts (title, 2 and 3):
# "T1" or "Note" is a container, not a concept (trial of 07.11).
SHEET_LEVEL = 4
# A run of spreadsheet rows becomes one block until it holds about this many characters; each further block starts
# with the run's first row again (most often its header), so a number far down a long table keeps its column names.
# On the 22 real workbooks of the mixed set, one run without a limit held 1,190 rows and 1.07 million characters.
ROW_BLOCK_CHARS = 4000
STYLE_HEADING = re.compile(r"^heading ([1-6])$")
class NoReader(Exception):
"""The reader this file type needs is missing or not the pinned version."""
class Unreadable(Exception):
"""The file is not a sound file of its type."""
@dataclass(frozen=True)
class Unit:
locator: dict[str, Any]
visible: str # what a reader of the document sees, whitespace collapsed
text: str # visible plus hidden content, whitespace collapsed
heading: int = 0 # the Markdown heading level the unit renders as, 0 for none
group: str = "" # units of one group render as one block (a run of spreadsheet rows)
line: str = "" # units of one line render as one line, each after its column letter (a spreadsheet row)
@dataclass(frozen=True)
class Rendering:
text: str
locators: list[dict[str, Any] | None] # the place of line N at index N-1, None for a blank line or a heading
left_out: int # units with hidden content, left out of the text
def collapse(text: str) -> str:
return SPACE.sub(" ", text).strip()
def xml(z: zipfile.ZipFile, part: str) -> ElementTree.Element:
try:
return ElementTree.fromstring(z.read(part))
except KeyError as e:
raise Unreadable(f"the package has no {part}") from e
except ElementTree.ParseError as e:
raise Unreadable(f"{part} is not XML: {e}") from e
def targets(z: zipfile.ZipFile, part: str) -> dict[str, str]:
"""Relationship id -> package part, for the relationships of part."""
folder, name = posixpath.split(part)
rels = posixpath.join(folder, "_rels", name + ".rels")
if rels not in z.namelist():
return {}
found: dict[str, str] = {}
for rel in xml(z, rels).iter(f"{PKG}Relationship"):
target = rel.get("Target", "")
if rel.get("TargetMode") == "External":
continue
found[rel.get("Id", "")] = (target[1:] if target.startswith("/")
else posixpath.normpath(posixpath.join(folder, target)))
return found
def walk(node: ElementTree.Element, skip: tuple[str, ...]) -> list[ElementTree.Element]:
"""node and its descendants in document order, without the subtrees of the tags in skip."""
out = [node]
for child in node:
if child.tag not in skip:
out += walk(child, skip)
return out
def docx_units(z: zipfile.ZipFile) -> list[Unit]:
levels: dict[str, int] = {}
if "word/styles.xml" in z.namelist():
for style in xml(z, "word/styles.xml").iter(f"{W}style"):
name = style.find(f"{W}name")
m = STYLE_HEADING.match((name.get(f"{W}val", "") if name is not None else "").lower())
if m or (name is not None and name.get(f"{W}val", "").lower() == "title"):
levels[style.get(f"{W}styleId", "")] = int(m.group(1)) if m else 1
body = xml(z, "word/document.xml").find(f"{W}body")
if body is None:
raise Unreadable("word/document.xml has no body")
box = (f"{W}txbxContent", MC_FALLBACK)
units: list[Unit] = []
for p in (e for e in walk(body, box) if e.tag == f"{W}p"):
visible: list[str] = []
full: list[str] = []
for run in (e for e in walk(p, box) if e.tag == f"{W}r"):
props = run.find(f"{W}rPr")
vanish = props.find(f"{W}vanish") if props is not None else None
hidden = vanish is not None and vanish.get(f"{W}val", "true") not in ("0", "false")
for e in run:
piece = (e.text or "" if e.tag in (f"{W}t", f"{W}delText") else
"\t" if e.tag == f"{W}tab" else " " if e.tag in (f"{W}br", f"{W}cr") else "")
full.append(piece)
if not hidden and e.tag != f"{W}delText":
visible.append(piece)
mark = p.find(f"{W}pPr/{W}pStyle")
level = levels.get(mark.get(f"{W}val", "") if mark is not None else "", 0)
units.append(Unit({"paragraph": len(units) + 1}, collapse("".join(visible)), collapse("".join(full)), level))
return units
def cell_text(cell: ElementTree.Element, shared: list[str]) -> str:
kind = cell.get("t", "n")
if kind == "inlineStr":
node = cell.find(f"{S}is")
return "" if node is None else string_item(node)
value = cell.find(f"{S}v")
raw = "" if value is None or value.text is None else value.text
if kind == "s":
try:
return shared[int(raw)]
except (ValueError, IndexError) as e:
raise Unreadable(f"cell {cell.get('r')} names shared string {raw!r}, which does not exist") from e
if kind == "b":
return {"1": "TRUE", "0": "FALSE"}.get(raw, raw)
return raw
def string_item(si: ElementTree.Element) -> str:
"""The text of a shared or inline string, rich runs joined, phonetic runs left out."""
return "".join(t.text or "" for t in walk(si, (f"{S}rPh",)) if t.tag == f"{S}t")
def letters(ref: str) -> str:
"""The column letters of a cell reference: "AB" of "AB12"."""
return re.match(r"[A-Za-z]*", ref).group(0) # type: ignore[union-attr]
def column(ref: str) -> int:
n = 0
for ch in letters(ref).upper():
n = n * 26 + ord(ch) - 64
return n
def xlsx_units(z: zipfile.ZipFile) -> list[Unit]:
shared = ([string_item(si) for si in xml(z, "xl/sharedStrings.xml").iter(f"{S}si")]
if "xl/sharedStrings.xml" in z.namelist() else [])
parts = targets(z, "xl/workbook.xml")
units: list[Unit] = []
for sheet in xml(z, "xl/workbook.xml").iter(f"{S}sheet"):
name = sheet.get("name", "")
part = parts.get(sheet.get(f"{R}id", ""))
if part is None:
raise Unreadable(f"sheet {name!r} has no worksheet part")
root = xml(z, part)
sheet_hidden = sheet.get("state", "visible") != "visible"
hidden_cols: set[int] = set()
for col in root.iter(f"{S}col"):
if col.get("hidden") in ("1", "true"):
hidden_cols.update(range(int(col.get("min", "0")), int(col.get("max", "0")) + 1))
first = True
block, last = 0, 0 # an empty row between two rows ends a block; a hidden row does not
for row in root.iter(f"{S}row"):
row_hidden = row.get("hidden") in ("1", "true")
number = int(row.get("r", "0") or "0")
shown = not row_hidden and any(collapse(cell_text(c, shared)) for c in row.iter(f"{S}c"))
if shown:
if number != last + 1:
block += 1
last = number
elif row_hidden and number == last + 1:
last = number
for cell in row.iter(f"{S}c"):
ref = cell.get("r", "")
text = collapse(cell_text(cell, shared))
if not text:
continue
hidden = sheet_hidden or row_hidden or column(ref) in hidden_cols
heading = name if first and not sheet_hidden else ""
if heading:
units.append(Unit({"sheet": name}, heading, heading, SHEET_LEVEL))
first = False
units.append(Unit({"sheet": name, "cell": ref}, "" if hidden else text, text, 0,
f"{name}\n{block}", f"{name}\n{row.get('r', '')}"))
return units
def slide_paragraphs(root: ElementTree.Element) -> list[tuple[str, bool]]:
"""(text, is the title) of every paragraph in a slide or notes part, in document order."""
out: list[tuple[str, bool]] = []
for shape in (e for e in walk(root, (MC_FALLBACK,)) if e.tag == f"{P}sp" or e.tag == f"{P}graphicFrame"):
ph = shape.find(f"{P}nvSpPr/{P}nvPr/{P}ph")
if ph is not None and ph.get("type") in ("sldNum", "sldImg", "dt", "ftr", "hdr"):
continue
title = ph is not None and ph.get("type") in ("title", "ctrTitle")
for para in shape.iter(f"{A}p"):
text = "".join((e.text or "") if e.tag == f"{A}t" else " " if e.tag == f"{A}br" else ""
for e in para.iter())
out.append((collapse(text), title))
return out
def pptx_units(z: zipfile.ZipFile) -> list[Unit]:
parts = targets(z, "ppt/presentation.xml")
units: list[Unit] = []
for n, sld in enumerate(xml(z, "ppt/presentation.xml").iter(f"{P}sldId"), 1):
part = parts.get(sld.get(f"{R}id", ""))
if part is None:
raise Unreadable(f"slide {n} has no part")
root = xml(z, part)
hidden = root.get("show") in ("0", "false")
for text, title in slide_paragraphs(root):
if text:
units.append(Unit({"slide": n, "notes": False}, "" if hidden else text, text, 2 if title else 0))
notes = [t for t in targets(z, part).values() if "/notesSlides/" in "/" + t]
for note in notes[:1]:
for text, _ in slide_paragraphs(xml(z, note)):
if text:
units.append(Unit({"slide": n, "notes": True}, "" if hidden else text, text))
return units
def pdf_units(path: Path) -> list[Unit]:
try:
import pypdf # type: ignore[import-not-found, unused-ignore]
except ImportError as e:
raise NoReader(f"{path.name}: reading PDF needs pypdf=={PDF_PIN} (python3 -m pip install pypdf=={PDF_PIN})") from e
logging.getLogger("pypdf").setLevel(logging.ERROR) # repairs of a sloppy file are not the user's business
if pypdf.__version__ != PDF_PIN:
raise NoReader(f"{path.name}: reading PDF needs pypdf=={PDF_PIN}, found {pypdf.__version__} "
f"(python3 -m pip install pypdf=={PDF_PIN})")
try:
doc = pypdf.PdfReader(str(path))
if doc.is_encrypted and not doc.decrypt(""):
raise Unreadable(f"{path.name} is encrypted")
texts = [page.extract_text() or "" for page in doc.pages]
except Unreadable:
raise
except Exception as e: # pypdf raises many kinds on a broken file
raise Unreadable(f"{path.name} is not a readable PDF: {e}") from e
return [Unit({"page": n}, collapse(t), collapse(t)) for n, t in enumerate(texts, 1)]
INVISIBLE = frozenset({"head", "title", "script", "style", "noscript", "template", "svg"})
class TextNodes(html.parser.HTMLParser):
"""Every visible text node of an HTML page, cut at its line breaks: (line of the raw source, text)."""
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.open: list[str] = []
self.found: list[tuple[int, str]] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if tag in INVISIBLE:
self.open.append(tag)
def handle_endtag(self, tag: str) -> None:
if tag in self.open:
del self.open[len(self.open) - 1 - self.open[::-1].index(tag):]
def handle_data(self, data: str) -> None:
if not self.open:
line = self.getpos()[0]
self.found += [(line + i, piece) for i, piece in enumerate(data.split("\n"))]
def html_units(path: Path) -> list[Unit]:
parser = TextNodes()
try:
parser.feed(path.read_text(encoding="utf-8"))
parser.close()
except UnicodeDecodeError as e:
raise Unreadable(f"{path.name} is not UTF-8") from e
return [Unit({"line": n}, collapse(t), collapse(t)) for n, t in parser.found if t.strip()]
def text_units(path: Path) -> list[Unit]:
try:
lines = path.read_text(encoding="utf-8").split("\n")
except UnicodeDecodeError as e:
raise Unreadable(f"{path.name} is not UTF-8") from e
return [Unit({"line": n}, collapse(t), collapse(t)) for n, t in enumerate(lines, 1)]
def units(path: Path) -> list[Unit]:
ext = path.suffix.lower()
if ext == ".pdf":
return pdf_units(path)
if ext in (".html", ".htm"):
return html_units(path)
if ext not in OFFICE:
return text_units(path)
try:
with zipfile.ZipFile(path) as z:
return {".docx": docx_units, ".xlsx": xlsx_units, ".pptx": pptx_units}[ext](z)
except zipfile.BadZipFile as e:
raise Unreadable(f"{path.name} is not an Office package: {e}") from e
def render(path: Path) -> Rendering:
"""The text a reader of the document sees, as Markdown the block splitter reads: a heading line per heading unit,
one line per unit, a blank line between blocks (a unit, or a run of spreadsheet rows). A spreadsheet row is one
line, each cell after its column letter ("A: Quantity | C: 101000"), so a number keeps the column its header
names; its place lists the cells with the offset each starts at (common.place_at). Hidden units are left out."""
lines: list[str] = []
places: list[dict[str, Any] | None] = []
left_out = 0
group: str | None = None
line: str | None = None
block, first_row = 0, 0 # where the current block, and the current run of rows, start in lines
for u in units(path):
if u.visible != u.text:
left_out += 1
if not u.visible:
continue
if u.line and u.line == line:
lines[-1] += " | "
places[-1]["cells"].append([u.locator["cell"], len(lines[-1])]) # type: ignore[index]
lines[-1] += f"{letters(u.locator['cell'])}: {u.visible}"
continue
same_run = bool(u.group and u.group == group)
if lines and not same_run:
lines.append("")
places.append(None)
block = first_row = len(lines)
elif same_run and u.line and sum(len(x) for x in lines[block:]) >= ROW_BLOCK_CHARS:
head, at = lines[first_row], places[first_row] or {}
lines += ["", head]
places += [None, {"sheet": at.get("sheet"), "cells": [list(c) for c in at.get("cells", [])]}]
block = len(lines) - 1
group, line = u.group or None, u.line or None
if u.line:
lines.append(f"{letters(u.locator['cell'])}: {u.visible}")
places.append({"sheet": u.locator["sheet"], "cells": [[u.locator["cell"], 0]]})
elif u.heading:
lines.append("#" * u.heading + " " + u.visible)
places.append(None)
else:
lines.append(("\\" + u.visible) if STRUCTURE.match(u.visible) else u.visible)
places.append(u.locator)
return Rendering("\n".join(lines) + "\n" if lines else "", places, left_out)
def locate(path: Path, quotes: list[str]) -> list[list[tuple[dict[str, Any], bool]]]:
"""For each quote, every (place, hidden) where it occurs, in the order of the document."""
found = units(path)
out: list[list[tuple[dict[str, Any], bool]]] = []
for quote in quotes:
q = collapse(quote)
places: list[tuple[dict[str, Any], bool]] = []
for u in found:
if "cell" not in u.locator and "sheet" in u.locator:
continue # a sheet's own heading unit is not a place in the workbook
if q and q in u.visible:
places.append((u.locator, False))
elif q and q in u.text:
places.append((u.locator, True))
out.append(places)
return out
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="python3 -m okfwiki.readers", description=__doc__.split("\n")[0])
sub = parser.add_subparsers(dest="command", required=True)
p = sub.add_parser("locate")
p.add_argument("file", type=Path)
p.add_argument("queries", type=Path)
args = parser.parse_args(argv)
try:
doc = json.loads(args.queries.read_text(encoding="utf-8"))
queries = [(q["id"], q["quote"]) for q in doc["queries"]]
answers = locate(args.file, [quote for _, quote in queries])
except (OSError, ValueError, KeyError, TypeError, NoReader, Unreadable) as e:
print(f"readers cannot run: {e}", file=sys.stderr)
return 2
print(json.dumps({"answers": [{"id": i, "found": [{"locator": loc, "hidden": h} for loc, h in places]}
for (i, _), places in zip(queries, answers)]}, ensure_ascii=False))
return 0
if __name__ == "__main__":
sys.exit(main())

569
okfwiki/serve.py Normal file
View file

@ -0,0 +1,569 @@
"""A read-only MCP server over one or more built collections (motor C21). It calls no model and writes no file.
serve --bundle NAME WIKI [SOURCES] [--bundle NAME WIKI [SOURCES] ...]
Speaks the MCP stdio transport: JSON-RPC 2.0, one message per line on stdin and stdout; anything else goes to stderr.
Register it with Claude Code as, for example,
`claude mcp add okf-wiki -- okf-wiki serve --bundle hage /path/wiki /path/sources`. Four tools, the skill's router for
answering a question:
collections each collection: its folders, pages, main pages, source files and okf_version
search words against each page's title, other names and description in index.md (SPEC section 8) and its
text (footnote lines and link targets left out); a page ranks by how rare its matching words are, a
word in the title or in another name of the page (`x-aliases`, at the end of its index line as
" (also: A; B)") counting twice;
a word matches the same word or one with an ending of at most two letters on a stem of four
(kmerge.word_match); main_only keeps main pages, and with no words lists them all, one per document
page one concept by its id (SPEC section 2: its path without .md): frontmatter, other names, sources with
quote and place, body, links out (each marked when it names no page) and the pages that link to it
source lines of the frozen copy of a source around a cited line, with the heading above, each line's page,
slide or cell, and the original file
Titles and descriptions come from index.md, and page text, main pages and inbound links from a plain scan of the
page files without parsing their YAML (which takes 67 s on wiki-38): the first call over wiki-38, 3 141 pages, reads
it in 2.3 s (08.11), and a search after that takes milliseconds. A collection is read again when its index.md or its
folder changes. Nothing outside a collection's folder or its copy of the sources is read. Plain file search in the
same folders works beside it.
"""
from __future__ import annotations
import argparse
import json
import math
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, TextIO
from okfwiki.audit import LINK, SCHEME, read_document, resolve
from okfwiki.common import place_words
from okfwiki.kmerge import name_words, word_match
from okfwiki.view import nearest_heading
PROTOCOL_VERSIONS = ("2025-11-25", "2025-06-18", "2025-03-26", "2024-11-05") # newest first
SERVER = {"name": "okf-wiki", "version": "1"}
RESERVED = ("index.md", "log.md") # SPEC 3.1
FOOTNOTE_DEF = re.compile(r"^\[\^[^\]]+\]:")
INDEX_LINE = re.compile(r"^[-*] \[(?P<title>.*?)\]\((?P<href>[^()\s]+?\.md)\)(?: - (?P<desc>.*))?$")
ALSO = re.compile(r"^(?P<desc>.*) \(also: (?P<names>.+)\)$") # a page's other names, as kassemble ends its index line
DEFAULT_LIMIT = 20
DEFAULT_WINDOW = 2
# Claude Code (2.1.319, 08.11) kept a tool result of 47 062 characters whole and put one of 55 819 in a file; a page
# longer than this comes in parts, each asked for with `part`.
PART_CHARS = 40_000
INSTRUCTIONS = (
"Read-only lookups in built okf-wiki collections. No call here runs a model, and nothing is written.\n"
"To answer a question: (1) `search` each collection's pages with the question's words, in the collection's own "
"words and language: it matches words, not meaning, and says how many pages each word hits and how many pages "
"matched of how many; (2) open the main page of the area with `page` (`search` with main_only lists the main "
"pages, one per document); (3) follow its links_out with `page`, and linked_from to see what builds on it; "
"(4) open the source at a footnote's line with `source` when the quote has to be checked or the page does not "
"say enough. Cite the page and the source (document and place). Say plainly what the collections do not cover. "
"Plain file search in the collection folders works beside this server.")
class ToolError(Exception):
"""A call the server cannot answer: reported to the caller as a tool result with isError."""
@dataclass(frozen=True)
class Collection:
name: str
wiki: Path
sources: Path | None
@dataclass
class Entry:
id: str
title: str
description: str
title_words: set[str] # of the title and of the page's other names
words: set[str] # title, other names, description and page text
aliases: list[str] = field(default_factory=list)
@dataclass
class Catalogue:
stamp: tuple[int, int, int]
okf_version: str | None
entries: list[Entry]
main_for: dict[str, list[str]]
linked_from: dict[str, list[str]]
vocabulary: dict[str, set[int]] = field(default_factory=dict) # word -> entry indexes
# --- reading a collection ----------------------------------------------------------------------------------------
def stamp(wiki: Path) -> tuple[int, int, int]:
index = (wiki / "index.md").stat()
return index.st_mtime_ns, index.st_size, wiki.stat().st_mtime_ns
def frontmatter_text(text: str) -> tuple[str, str]:
"""(frontmatter, body) of a page as plain text, split at the closing --- (SPEC 4), without parsing the YAML."""
lines = text.split("\n")
if not lines or lines[0].strip() != "---":
return "", text
end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
if end is None:
return "", text
return "\n".join(lines[1:end]), "\n".join(lines[end + 1:])
def main_for_of(front: str) -> list[str]:
"""The sources a page is the main page of (`x-main-for`), read from the frontmatter text: a block list as
kassemble writes it, or one value on the key's line."""
out: list[str] = []
lines = front.split("\n")
for i, raw in enumerate(lines):
if not raw.startswith("x-main-for:"):
continue
rest = raw[len("x-main-for:"):].strip()
if rest:
out.extend(v.strip().strip("'\"") for v in rest.strip("[]").split(",") if v.strip())
for item in lines[i + 1:]:
if not item.startswith("- "):
break
out.append(item[2:].strip().strip("'\""))
return out
def concept_id(rel: str) -> str:
return rel[:-3] if rel.endswith(".md") else rel
def load_catalogue(wiki: Path) -> Catalogue:
st = stamp(wiki)
index_text = (wiki / "index.md").read_text(encoding="utf-8")
front, body = frontmatter_text(index_text)
version = next((ln.split(":", 1)[1].strip().strip("'\"") for ln in front.split("\n")
if ln.startswith("okf_version:")), None)
entries: list[Entry] = []
seen: set[str] = set()
for raw in body.split("\n"):
m = INDEX_LINE.match(raw.strip())
if not m:
continue
rel = resolve("index.md", m.group("href"))
if not rel or rel in seen or rel.split("/")[-1] in RESERVED:
continue
seen.add(rel)
title, desc = m.group("title"), m.group("desc") or ""
also = ALSO.match(desc)
aliases = [a.strip() for a in also.group("names").split("; ") if a.strip()] if also else []
desc = also.group("desc") if also else desc
tw = set(name_words(" ".join([title, *aliases])))
entries.append(Entry(concept_id(rel), title, desc, tw, tw | set(name_words(desc)), aliases))
main_for: dict[str, list[str]] = {}
linked: dict[str, set[str]] = {}
body_words: dict[str, set[str]] = {}
for p in sorted(wiki.rglob("*.md")):
rel = p.relative_to(wiki).as_posix()
if p.name in RESERVED or not p.is_file():
continue
try:
page_front, page_body = frontmatter_text(p.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError):
continue
mains = main_for_of(page_front)
if mains:
main_for[concept_id(rel)] = mains
text = "\n".join(ln for ln in page_body.split("\n") if not FOOTNOTE_DEF.match(ln))
body_words[concept_id(rel)] = set(name_words(LINK.sub(lambda m: m.group(1), text)))
for m in LINK.finditer(page_body):
target = m.group(2)
if not target or target.startswith("#") or SCHEME.match(target):
continue
to = resolve(rel, target)
if to and to.endswith(".md") and to != rel:
linked.setdefault(concept_id(to), set()).add(concept_id(rel))
cat = Catalogue(st, version, entries, main_for, {k: sorted(v) for k, v in linked.items()})
for n, e in enumerate(entries):
e.words |= body_words.get(e.id, set())
for w in e.words:
cat.vocabulary.setdefault(w, set()).add(n)
return cat
# --- the server --------------------------------------------------------------------------------------------------
def schema(props: dict[str, Any], required: list[str]) -> dict[str, Any]:
return {"type": "object", "properties": props, "required": required, "additionalProperties": False}
COLLECTION_ARG = {"type": "string", "description": "The collection's name, as `collections` lists it."}
TOOLS: list[dict[str, Any]] = [
{"name": "collections",
"description": "List the collections this server reads: name, folders, pages, main pages, source files.",
"inputSchema": schema({}, [])},
{"name": "search",
"description": ("Search the pages of one or all collections (title, other names, description and text) for words. "
"Matches words, not meaning: use the collection's own words and language, and ask again with "
"other words when the counts say a word hits nothing. main_only keeps main pages (one per "
"document); main_only with no words lists them all."),
"inputSchema": schema({
"words": {"type": "string", "description": "The words to look for, separated by spaces."},
"collection": COLLECTION_ARG,
"main_only": {"type": "boolean", "description": "Only main pages (default false)."},
"limit": {"type": "integer", "minimum": 1, "description": f"Results per collection (default {DEFAULT_LIMIT})."},
"offset": {"type": "integer", "minimum": 0, "description": "Results to skip, for the next page (default 0)."},
}, [])},
{"name": "page",
"description": ("One concept page by its id (its file name without .md): title, description, sources with the "
"verbatim quote and its place, the body with footnotes, links out and the pages linking to it. "
"A long page comes in parts: `parts` says how many, ask for the next with `part`."),
"inputSchema": schema({"collection": COLLECTION_ARG,
"id": {"type": "string", "description": "The concept id, e.g. from search."},
"part": {"type": "integer", "minimum": 1, "description": "Which part (default 1)."}},
["collection", "id"])},
{"name": "source",
"description": ("Lines of a source's frozen copy around a cited line, with the heading above, each line's page, "
"slide or cell, and the original file. resource and line are a page source's resource and line."),
"inputSchema": schema({
"collection": COLLECTION_ARG,
"resource": {"type": "string", "description": "The source's resource, e.g. /kilde/report.pdf.md."},
"line": {"type": "integer", "minimum": 1, "description": "The cited line (1-based)."},
"before": {"type": "integer", "minimum": 0, "description": f"Lines before it (default {DEFAULT_WINDOW})."},
"after": {"type": "integer", "minimum": 0, "description": f"Lines after it (default {DEFAULT_WINDOW})."},
}, ["collection", "resource", "line"])},
]
for _tool in TOOLS:
_tool["annotations"] = {"readOnlyHint": True, "destructiveHint": False, "idempotentHint": True,
"openWorldHint": False}
def int_arg(args: dict[str, Any], key: str, default: int, least: int) -> int:
value = args.get(key, default)
if type(value) is not int or value < least:
raise ToolError(f"{key} must be a whole number of at least {least}")
return int(value)
class Server:
def __init__(self, collections: list[Collection]) -> None:
self.collections = {c.name: c for c in collections}
self.catalogues: dict[str, Catalogue] = {}
# protocol
def handle(self, msg: Any) -> dict[str, Any] | None:
if not isinstance(msg, dict) or msg.get("jsonrpc") != "2.0" or not isinstance(msg.get("method"), str):
return error(msg.get("id") if isinstance(msg, dict) else None, -32600, "not a JSON-RPC 2.0 request")
if "id" not in msg:
return None # a notification
mid, method, params = msg["id"], msg["method"], msg.get("params") or {}
if method == "initialize":
asked = params.get("protocolVersion") if isinstance(params, dict) else None
version = asked if asked in PROTOCOL_VERSIONS else PROTOCOL_VERSIONS[0]
return reply(mid, {"protocolVersion": version, "capabilities": {"tools": {"listChanged": False}},
"serverInfo": SERVER, "instructions": INSTRUCTIONS})
if method == "ping":
return reply(mid, {})
if method == "tools/list":
return reply(mid, {"tools": TOOLS})
if method == "tools/call":
name = params.get("name") if isinstance(params, dict) else None
args = params.get("arguments") or {} if isinstance(params, dict) else {}
tool = getattr(self, f"tool_{name}", None) if name in {t["name"] for t in TOOLS} else None
if tool is None or not isinstance(args, dict):
return error(mid, -32602, f"unknown tool {name!r}" if tool is None else "arguments must be an object")
try:
data, is_error = tool(args), False
except ToolError as exc:
data, is_error = {"error": str(exc)}, True
text = dumps(data)
return reply(mid, {"content": [{"type": "text", "text": text}], "isError": is_error})
return error(mid, -32601, f"method not found: {method}")
# collections
def collection(self, name: Any) -> Collection:
if name not in self.collections:
raise ToolError(f"no collection {name!r}; this server reads {', '.join(sorted(self.collections))}")
return self.collections[str(name)]
def catalogue(self, c: Collection) -> Catalogue:
try:
current = stamp(c.wiki)
cat = self.catalogues.get(c.name)
if cat is None or cat.stamp != current:
cat = self.catalogues[c.name] = load_catalogue(c.wiki)
except (OSError, UnicodeDecodeError) as exc:
raise ToolError(f"collection {c.name} cannot be read: {exc}") from exc
return cat
def tool_collections(self, args: dict[str, Any]) -> dict[str, Any]:
out = []
for c in self.collections.values():
cat = self.catalogue(c)
files = None
if c.sources is not None:
files = sum(1 for p in (c.sources / "kilde").rglob("*") if p.is_file())
out.append({"collection": c.name, "wiki": str(c.wiki.resolve()),
"sources": str(c.sources.resolve()) if c.sources else None,
"pages": len(cat.entries), "main_pages": len(cat.main_for), "source_files": files,
"okf_version": cat.okf_version})
return {"collections": out}
# search
def tool_search(self, args: dict[str, Any]) -> dict[str, Any]:
words = args.get("words", "")
if not isinstance(words, str):
raise ToolError("words must be a string")
main_only = args.get("main_only", False) is True
limit = int_arg(args, "limit", DEFAULT_LIMIT, 1)
offset = int_arg(args, "offset", 0, 0)
asked = list(dict.fromkeys(name_words(words)))
if not asked and not main_only:
raise ToolError("give words to search for, or main_only to list the main pages")
chosen = [self.collection(args["collection"])] if "collection" in args else list(self.collections.values())
return {"collections": [self.search_one(c, asked, main_only, limit, offset) for c in chosen]}
def search_one(self, c: Collection, asked: list[str], main_only: bool, limit: int, offset: int) -> dict[str, Any]:
cat = self.catalogue(c)
n = len(cat.entries)
hits: dict[int, list[str]] = {}
counts = []
weight: dict[str, float] = {}
for word in asked:
found: set[int] = set()
for known, at in cat.vocabulary.items():
if word_match(word, known):
found |= at
counts.append({"word": word, "pages": len(found)})
weight[word] = math.log((n + 1) / (len(found) + 0.5))
for i in found:
hits.setdefault(i, []).append(word)
if asked:
pool = list(hits)
else:
pool = list(range(n))
results = []
for i in pool:
e = cat.entries[i]
if main_only and e.id not in cat.main_for:
continue
matched = hits.get(i, [])
score = sum(weight[w] * (2 if any(word_match(w, t) for t in e.title_words) else 1) for w in matched)
results.append((-score, e.title.casefold(), e.id, e, matched, score))
results.sort(key=lambda r: (r[0], r[1], r[2]))
page = results[offset:offset + limit]
return {"collection": c.name, "pages": n, "matched": len(results), "words": counts, "offset": offset,
"more": offset + limit < len(results),
"results": [{"id": e.id, "title": e.title, "aliases": e.aliases, "description": e.description,
"main_for": cat.main_for.get(e.id, []), "score": round(score, 3), "words": matched}
for _, _, _, e, matched, score in page]}
# page
def page_rel(self, c: Collection, pid: Any) -> str:
if not isinstance(pid, str) or not pid.strip():
raise ToolError("id must be a concept id, e.g. from search")
rel = pid.strip()
rel = rel if rel.endswith(".md") else rel + ".md"
norm = resolve("index.md", rel.lstrip("/")) if not rel.startswith("/") else None
path = (c.wiki / norm) if norm else None
if (not norm or norm.split("/")[-1] in RESERVED or path is None or not path.is_file()
or not path.resolve().is_relative_to(c.wiki.resolve())):
raise ToolError(f"no concept {pid!r} in {c.name}; find its id with search")
return norm
def tool_page(self, args: dict[str, Any]) -> dict[str, Any]:
c = self.collection(args.get("collection"))
cat = self.catalogue(c)
rel = self.page_rel(c, args.get("id"))
try:
doc = read_document(c.wiki / rel, rel)
except (OSError, UnicodeDecodeError) as exc:
raise ToolError(f"{rel} cannot be read: {exc}") from exc
fm = doc.frontmatter or {}
srcs = fm.get("sources", [])
sources = []
for s in srcs if isinstance(srcs, list) else [srcs]:
if not isinstance(s, dict):
continue
raw_loc = s.get("x-locator")
loc: dict[str, Any] = raw_loc if isinstance(raw_loc, dict) else {}
sources.append({"id": s.get("id"), "resource": s.get("resource"), "title": s.get("title"),
"line": loc.get("line"), "place": place_words(loc) if loc else None,
"quote": s.get("x-quote"), "provenance": s.get("x-provenance")})
out_links: list[dict[str, Any]] = []
seen: set[str] = set()
for m in LINK.finditer(doc.body):
target = m.group(2)
if not target or target.startswith("#") or SCHEME.match(target):
continue
to = resolve(rel, target)
if not to or not to.endswith(".md") or to == rel or to in seen:
continue
seen.add(to)
out_links.append({"id": concept_id(to), "exists": (c.wiki / to).is_file()})
mains = fm.get("x-main-for", [])
names = fm.get("x-aliases", [])
head = {"collection": c.name, "id": concept_id(rel), "path": str((c.wiki / rel).resolve()),
"title": fm.get("title"), "aliases": names if isinstance(names, list) else [names],
"description": fm.get("description"), "type": fm.get("type"),
"main_for": mains if isinstance(mains, list) else [mains],
"links_out": out_links, "linked_from": cat.linked_from.get(concept_id(rel), [])}
parts = split_parts(head, doc.body.strip(), sources)
part = int_arg(args, "part", 1, 1)
if part > len(parts):
raise ToolError(f"{concept_id(rel)} has {len(parts)} part(s), not {part}")
return parts[part - 1]
# source
def tool_source(self, args: dict[str, Any]) -> dict[str, Any]:
c = self.collection(args.get("collection"))
if c.sources is None:
raise ToolError(f"collection {c.name} was started without its sources folder; give it as the third "
f"value of --bundle")
res = args.get("resource")
line = int_arg(args, "line", 0, 1)
before = int_arg(args, "before", DEFAULT_WINDOW, 0)
after = int_arg(args, "after", DEFAULT_WINDOW, 0)
rel = resolve("index.md", res) if isinstance(res, str) and res and not SCHEME.match(res) else None
root = c.sources.resolve()
path = (c.sources / rel) if rel else None
if (not rel or not rel.startswith("kilde/") or path is None or not path.is_file()
or not path.resolve().is_relative_to(root)):
raise ToolError(f"no source {res!r} in the sources of {c.name}; use a resource a page's sources name")
try:
lines = path.read_text(encoding="utf-8").splitlines()
except (OSError, UnicodeDecodeError) as exc:
raise ToolError(f"{rel} cannot be read: {exc}") from exc
if line > len(lines):
raise ToolError(f"{res} has {len(lines)} lines, not {line}")
entry = self.manifest_entry(c, rel)
places = entry.get("places") if isinstance(entry.get("places"), list) else None
shown = []
for n in range(max(1, line - before), min(len(lines), line + after) + 1):
item: dict[str, Any] = {"line": n, "text": lines[n - 1]}
place = places[n - 1] if places is not None and n - 1 < len(places) else None
if isinstance(place, dict):
item["place"] = place_words(place)
shown.append(item)
heading = nearest_heading(lines, line)
out: dict[str, Any] = {"collection": c.name, "resource": res, "path": str(path.resolve()),
"lines_total": len(lines), "lines": shown,
"heading": {"line": heading[0], "text": heading[1]} if heading else None}
if isinstance(entry.get("original"), str):
original: dict[str, Any] = {"name": Path(entry["original"]).name}
if isinstance(entry.get("url"), str):
original["url"] = entry["url"]
copy = entry.get("original_copy")
if isinstance(copy, str) and (c.sources / copy).resolve().is_relative_to(root):
original["copy"] = str((c.sources / copy).resolve())
out["original"] = original
elif isinstance(entry.get("url"), str):
out["original"] = {"url": entry["url"]}
return out
def manifest_entry(self, c: Collection, rel: str) -> dict[str, Any]:
assert c.sources is not None
try:
doc = json.loads((c.sources / "source-manifest.json").read_text(encoding="utf-8"))
except (OSError, ValueError):
return {}
for e in doc.get("sources", []) if isinstance(doc, dict) else []:
if isinstance(e, dict) and e.get("path") == rel:
return e
return {}
def dumps(data: Any) -> str:
return json.dumps(data, ensure_ascii=False, separators=(",", ":"))
def split_parts(head: dict[str, Any], body: str, sources: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""A page as parts of at most PART_CHARS characters of JSON each: the first carries the page's fields, every part
its share of the body (whole paragraphs, in order) and then of the sources (whole entries, in order). The bodies
joined with a blank line are the page's body, and nothing is left out; one paragraph or entry longer than a part
stands alone in its part."""
items: list[tuple[str, Any]] = [("p", p) for p in body.split("\n\n")] if body else []
items += [("s", s) for s in sources]
def build(base: dict[str, Any], chunk: list[tuple[str, Any]], k: int, n: int) -> dict[str, Any]:
return {**base, "part": k, "parts": n, "body": "\n\n".join(v for t, v in chunk if t == "p"),
"sources": [v for t, v in chunk if t == "s"]}
chunks: list[list[tuple[str, Any]]] = [[]]
for item in items:
base = head if len(chunks) == 1 else {"collection": head["collection"], "id": head["id"]}
trial = chunks[-1] + [item]
if chunks[-1] and len(dumps(build(base, trial, 99999, 99999))) > PART_CHARS:
chunks.append([item])
else:
chunks[-1] = trial
n = len(chunks)
return [build(head if k == 1 else {"collection": head["collection"], "id": head["id"]}, chunk, k, n)
for k, chunk in enumerate(chunks, 1)]
def reply(mid: Any, result: dict[str, Any]) -> dict[str, Any]:
return {"jsonrpc": "2.0", "id": mid, "result": result}
def error(mid: Any, code: int, message: str) -> dict[str, Any]:
return {"jsonrpc": "2.0", "id": mid, "error": {"code": code, "message": message}}
def run(server: Server, inp: TextIO, out: TextIO) -> None:
for raw in inp:
if not raw.strip():
continue
try:
msg = json.loads(raw)
except ValueError:
answer: dict[str, Any] | None = error(None, -32700, "not JSON")
else:
answer = server.handle(msg)
if answer is not None:
out.write(json.dumps(answer, ensure_ascii=False) + "\n")
out.flush()
def collections_from(groups: list[list[str]]) -> list[Collection]:
out: list[Collection] = []
for g in groups:
if len(g) not in (2, 3):
raise ValueError(f"--bundle takes NAME WIKI [SOURCES], not {' '.join(g)}")
name, wiki = g[0], Path(g[1]).expanduser()
sources = Path(g[2]).expanduser() if len(g) == 3 else None
if any(c.name == name for c in out):
raise ValueError(f"two collections are named {name}")
if not (wiki / "index.md").is_file():
raise ValueError(f"{wiki} has no index.md, so it is not a built collection")
if sources is not None and not (sources / "source-manifest.json").is_file():
raise ValueError(f"{sources} has no source-manifest.json, so it is not a sources folder")
out.append(Collection(name, wiki, sources))
return out
def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(prog="okf-wiki serve", description=__doc__.split("\n")[0])
ap.add_argument("--bundle", action="append", nargs="+", required=True, metavar="NAME WIKI [SOURCES]",
help="a collection to serve: its name, its folder, and its sources folder (repeat for more)")
try:
ns = ap.parse_args(argv)
except SystemExit as exc:
return int(exc.code or 0) if isinstance(exc.code, int) else 2
try:
collections = collections_from(ns.bundle)
except ValueError as exc:
print(f"serve: {exc}", file=sys.stderr)
return 2
print(f"okf-wiki serve: {', '.join(c.name for c in collections)}", file=sys.stderr)
run(Server(collections), sys.stdin, sys.stdout)
return 0
if __name__ == "__main__":
sys.exit(main())

830
okfwiki/sources.py Normal file
View file

@ -0,0 +1,830 @@
"""The source manifest of a frozen copy (contract K1): check, freeze and subset; the change warning (contract K4): diff
and marks. No network, no model.
Usage:
python3 -m okfwiki.sources check ROOT
python3 -m okfwiki.sources freeze STAGING --meta FILE --out ROOT [--frozen-at YYYY-MM-DDTHH:MM:SSZ]
python3 -m okfwiki.sources folder DIR --collection NAME --out ROOT [--frozen-at YYYY-MM-DDTHH:MM:SSZ]
python3 -m okfwiki.sources subset ROOT --paths FILE --out ROOT2 [--frozen-at YYYY-MM-DDTHH:MM:SSZ]
python3 -m okfwiki.sources diff OLD NEW
python3 -m okfwiki.sources marks COLLECTION --diff FILE
python3 -m okfwiki.sources reread ROOT --out ROOT2 [--frozen-at YYYY-MM-DDTHH:MM:SSZ]
python3 -m okfwiki.sources watch ROOT --into DIR [--bundle COLLECTION] [--frozen-at YYYY-MM-DDTHH:MM:SSZ]
A frozen copy is a root folder with source-manifest.json and the source files under kilde/. A page cites a source as
sources[].resource "/" + path.
check Exit 0 when every listed file exists with the same sha256, bytes and lines, every file under kilde/ is
listed, and the manifest has the form of K1; exit 1 with the problems listed; exit 2 when it cannot run (no
manifest, not JSON, another schema, sources not a list).
freeze Copies the files of STAGING (flat, named as the part of path after kilde/) to ROOT/kilde/, makes them read
only and writes the manifest. The meta file says per file what freeze cannot know:
{"collection": "...", "index": {...} (optional), "schema": "okf-wiki/source-meta/1",
"sources": {"<name>": {"kind": "...", "main_topic": true|false, "url": "...", "version": "..."}}}
Refused with exit 1 and the files listed: a file the meta lacks or a meta entry without a file, an invalid
entry, a file that is not UTF-8, and a file whose lines differ between splitlines() and split("\\n") (the
auditor uses the one, the trial code the other, so line N must be the same for both). Exit 2 when ROOT exists.
folder Freezes a folder of documents as it is, with no staging and no hand-written meta: every file under DIR, in
subfolders too, whose suffix has a reader (.md, .markdown, .txt, and .pdf, .docx, .xlsx, .pptx through
okfwiki.readers). A PDF or Office file is frozen as the text a reader of it sees (its safe name plus ".md"),
with the file itself copied read-only to original/ (`original_copy`), `reader` (the type), `original_sha256`,
`places` (the page, paragraph, cell or slide of each line, null
for a blank line or a heading) and `left_out` (hidden parts left out, also reported); every document records
`lang` (common.language) when its words tell it, for the view; one that cannot be read
or has no text is skipped and named, and a PDF without pypdf at the pinned version is refused with exit 1
and the install command. Each gets a safe name under kilde/: its path
relative to DIR with "/" as "__", letters spelled in ASCII (æ ø å as ae oe aa, accents dropped), lower case,
any other character as "-"; a name that would read as a release note (release-vX.Y.Z.md) ends in "-doc.md"
instead, since the release-note rules go by the name. The entry keeps the path as it was (`original`, NFC), the file's own URL (`url`,
file:///) and what was changed to make it a source (`normalised`, [] when the copy is byte for byte): "bom" (a
UTF-8 byte order mark dropped), "cp1252" (decoded as Windows-1252, the Western Latin-1 superset, when not
UTF-8) and "line-breaks" (\r\n, \r, form feed and every other line break written as \n), so a quote
stands verbatim in the frozen copy. Kind "document", one main topic. A file is skipped, and named with the
reason, when it is hidden (a name starting with "."; a hidden folder is skipped whole), has no reader, is
empty or is a link. Two files with one safe name are refused with exit 1; exit 2 when no file is left, or DIR
is not a folder, or ROOT exists. The manifest names the folder read (`folder`, its resolved path), so the copy
can be read again from itself.
subset A part of ROOT with the listed paths (one kilde/... per line, sorted, no duplicates), its own frozen_at, the
index unchanged and subset_of pointing at ROOT's frozen_at and the sha256 of ROOT's manifest. Exit 2 when ROOT
fails check, the paths file is not as described, or ROOT2 exists.
diff Which sources of NEW are new, changed or gone since OLD, by path. Changed = another sha256 or another kind (the
kind decides what a source is to the build). Exit 0 = no change, 1 = at least one, 2 = cannot run (a root
fails check, or the two roots name another collection).
marks The pages of COLLECTION (every .md under it, named by its path with "/") that cite a changed or gone source
of the diff as sources[].resource "/" + path. New sources mark nothing. A page without frontmatter cites
nothing; one whose frontmatter cannot be read stops marks. Exit 0 = no page marked, 1 = at least one, 2 =
cannot run.
Both print their report with the one written form, so two runs are byte-identical. A source read from a folder
is named in diff with its `original` path too, as the user knows the file.
reread Reads the folder ROOT was read from again, as the same collection, into ROOT2 (the folder command's rules). The
folder is the manifest's `folder`, or for a copy frozen before that field, the one folder its entries name
(each url less its original path). Exit 2 when ROOT fails check, was not read from a folder, its entries name
two folders, the folder is gone, or ROOT2 exists.
watch The change warning for a folder, one command for any scheduler (motor B15): reads the folder of ROOT again into
DIR/latest (replaced whole each run), writes diff.json there and, with --bundle, marks.json, and prints each
difference by its original path with the pages marked on it. DIR/watch-state.json keeps the lines reported,
so a run says only what is new since the last one (a new frozen copy starts over). Exit 0 = nothing new,
1 = something new, 2 = cannot run (also DIR inside the watched folder: the next reading would take the
watch's files for documents). Only a warning: the pages are rebuilt from DIR/latest in a session.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import shutil
import sys
import tempfile
import unicodedata
import urllib.parse
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import yaml
from okfwiki import readers
from okfwiki.common import RunError, ascii_fold, language
from okfwiki.kblocks import RELEASE
SCHEMA = "okf-wiki/source-manifest/1"
META_SCHEMA = "okf-wiki/source-meta/1"
DIFF_SCHEMA = "okf-wiki/source-diff/1"
MARKS_SCHEMA = "okf-wiki/source-marks/1"
MANIFEST = "source-manifest.json"
KILDE = "kilde"
PATH = re.compile(r"^kilde/[a-z0-9._-]+$")
NAME = re.compile(r"^[a-z0-9._-]+$")
HEX64 = re.compile(r"^[0-9a-f]{64}$")
FROZEN_AT = re.compile(r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$")
READ_ONLY = 0o444
class Refused(Exception):
"""The input breaks a rule; nothing was written. A command reports the problems with exit 1."""
def __init__(self, problems: list[str]) -> None:
super().__init__("; ".join(problems))
self.problems = problems
def dump(data: Any) -> str:
"""The one written form of every file here, so two writings of the same content are byte-identical."""
return json.dumps(data, indent=1, sort_keys=True, ensure_ascii=False) + "\n"
def sha256(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def ambiguous_lines(text: str) -> bool:
"""True when splitlines() and split("\\n") (less a final empty part) give different lines."""
parts = text.split("\n")
if parts[-1] == "":
parts.pop()
return parts != text.splitlines()
def utc_now() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def load_json(path: Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError:
raise RunError(f"{path} does not exist") from None
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
raise RunError(f"{path} is not JSON: {exc}") from None
def load_manifest(root: Path) -> dict[str, Any]:
"""The manifest of ROOT, or RunError when there is none that check can read."""
if not root.is_dir():
raise RunError(f"{root} is not a directory")
doc = load_json(root / MANIFEST)
if not isinstance(doc, dict):
raise RunError(f"{root / MANIFEST} is not a JSON object")
if doc.get("schema") != SCHEMA:
raise RunError(f"{root / MANIFEST} has schema {doc.get('schema')!r}, not {SCHEMA}")
if not isinstance(doc.get("sources"), list):
raise RunError(f"{root / MANIFEST} has no list of sources")
return doc
def nonempty_str(value: Any) -> bool:
return isinstance(value, str) and value != ""
def describe_problems(meta: Any) -> list[str]:
"""The rules for what the meta says about one source (K1 rules 3, 4, 6)."""
if not isinstance(meta, dict):
return ["is not an object"]
out = []
if not nonempty_str(meta.get("kind")):
out.append("kind is not a non-empty string")
if not isinstance(meta.get("main_topic"), bool):
out.append(f"main_topic is {meta.get('main_topic')!r}, not true or false")
if not nonempty_str(meta.get("url")):
out.append("url is not a non-empty string")
if meta.get("kind") == "release-notes" and not nonempty_str(meta.get("version")):
out.append("release-notes has no version")
if "version" in meta and not nonempty_str(meta["version"]):
out.append("version is not a non-empty string")
return out
def index_problem(value: Any) -> str | None:
if (isinstance(value, dict) and isinstance(value.get("pages"), int) and not isinstance(value["pages"], bool)
and isinstance(value.get("sha256"), str) and HEX64.match(value["sha256"]) and nonempty_str(value.get("url"))):
return None
return f"index is {value!r}, not {{pages, sha256, url}}"
def subset_of_problem(value: Any) -> str | None:
if value is None:
return None
if (isinstance(value, dict) and isinstance(value.get("frozen_at"), str) and FROZEN_AT.match(value["frozen_at"])
and isinstance(value.get("manifest_sha256"), str) and HEX64.match(value["manifest_sha256"])):
return None
return f"subset_of is {value!r}, not null or {{frozen_at, manifest_sha256}}"
def file_facts(data: bytes) -> tuple[dict[str, Any], str | None]:
"""sha256, bytes and lines of a source file, and the reason it cannot be a source, if any."""
facts: dict[str, Any] = {"bytes": len(data), "sha256": sha256(data)}
try:
text = data.decode("utf-8")
except UnicodeDecodeError:
return facts, "is not UTF-8"
facts["lines"] = len(text.splitlines())
if ambiguous_lines(text):
return facts, "has ambiguous lines (a line break other than \\n)"
return facts, None
def check(root: Path) -> list[str]:
"""K1 rule 11 and the form of the manifest; [] when the root is sound. RunError when check cannot run."""
doc = load_manifest(root)
problems: list[str] = []
if not nonempty_str(doc.get("collection")):
problems.append("collection is not a non-empty string")
if not (isinstance(doc.get("frozen_at"), str) and FROZEN_AT.match(doc["frozen_at"])):
problems.append(f"frozen_at is {doc.get('frozen_at')!r}, not YYYY-MM-DDTHH:MM:SSZ in UTC")
if "index" in doc and (p := index_problem(doc["index"])):
problems.append(p)
if "folder" in doc and not (nonempty_str(doc["folder"]) and Path(doc["folder"]).is_absolute()):
problems.append(f"folder is {doc['folder']!r}, not the absolute path of the folder the copy was read from")
if "subset_of" not in doc:
problems.append("subset_of is missing (null for a full copy)")
elif p := subset_of_problem(doc["subset_of"]):
problems.append(p)
listed: set[str] = set()
paths: list[str] = []
for n, e in enumerate(doc["sources"], 1):
if not isinstance(e, dict) or not isinstance(e.get("path"), str):
problems.append(f"sources entry {n} has no path")
continue
path = e["path"]
paths.append(path)
if not PATH.match(path):
problems.append(f"{path}: path is not kilde/ followed by [a-z0-9._-]")
continue
if path in listed:
problems.append(f"{path}: listed more than once")
continue
listed.add(path)
problems += [f"{path}: {p}" for p in describe_problems(e)]
f = root / path
if not f.is_file() or f.is_symlink():
problems.append(f"{path}: missing")
continue
facts, why = file_facts(f.read_bytes())
for key in ("sha256", "bytes", "lines"):
if key in facts and e.get(key) != facts[key]:
problems.append(f"{path}: {key} is {facts[key]!r} on disk, {e.get(key)!r} in the manifest")
if why:
problems.append(f"{path}: {why}")
if "places" in e and not (isinstance(e["places"], list) and len(e["places"]) == facts.get("lines")):
problems.append(f"{path}: places do not name one place per line ({facts.get('lines')} lines)")
if "original_copy" in e:
copy = e["original_copy"]
c = root / copy if isinstance(copy, str) and re.fullmatch(rf"{ORIGINAL}/[a-z0-9._-]+", copy) else None
if c is None:
problems.append(f"{path}: original_copy {copy!r} is not {ORIGINAL}/ followed by [a-z0-9._-]")
elif not c.is_file() or c.is_symlink():
problems.append(f"{copy}: missing")
elif sha256(c.read_bytes()) != e.get("original_sha256"):
problems.append(f"{copy}: sha256 differs from original_sha256")
if paths != sorted(paths):
problems.append("sources are not sorted on path")
kilde = root / KILDE
if kilde.is_dir():
for f in sorted(kilde.rglob("*")):
rel = f.relative_to(root).as_posix()
if (f.is_file() or f.is_symlink()) and rel not in listed:
problems.append(f"{rel}: under kilde/ but not listed")
return problems
def write_root(out: Path, files: dict[str, bytes], doc: dict[str, Any], others: dict[str, bytes] | None = None) -> None:
"""Write a root in a temporary folder beside OUT and move it into place, so a failure leaves no half root. files
go under kilde/; others (relative path -> bytes) beside it, such as the original copies under original/."""
if out.exists():
raise RunError(f"{out} exists; refusing to overwrite")
out.parent.mkdir(parents=True, exist_ok=True)
tmp = Path(tempfile.mkdtemp(prefix=f".{out.name}-", dir=out.parent))
try:
(tmp / KILDE).mkdir()
for name, data in files.items():
p = tmp / KILDE / name
p.write_bytes(data)
os.chmod(p, READ_ONLY)
for rel, data in (others or {}).items():
p = tmp / rel
p.parent.mkdir(parents=True, exist_ok=True)
p.write_bytes(data)
os.chmod(p, READ_ONLY)
(tmp / MANIFEST).write_text(dump(doc), encoding="utf-8")
problems = check(tmp)
if problems:
raise Refused(problems)
tmp.rename(out)
except BaseException:
shutil.rmtree(tmp, ignore_errors=True)
raise
def checked_frozen_at(value: str | None) -> str:
frozen_at = value or utc_now()
if not FROZEN_AT.match(frozen_at):
raise RunError(f"--frozen-at {frozen_at!r} is not YYYY-MM-DDTHH:MM:SSZ")
return frozen_at
def freeze(staging: Path, meta_path: Path, out: Path, frozen_at: str | None = None) -> dict[str, Any]:
when = checked_frozen_at(frozen_at)
if out.exists():
raise RunError(f"{out} exists; refusing to overwrite")
if not staging.is_dir():
raise RunError(f"{staging} is not a directory")
meta = load_json(meta_path)
if not isinstance(meta, dict) or meta.get("schema") != META_SCHEMA or not isinstance(meta.get("sources"), dict):
raise RunError(f"{meta_path} is not a {META_SCHEMA} file with an object of sources")
described: dict[str, Any] = meta["sources"]
problems: list[str] = []
if not nonempty_str(meta.get("collection")):
problems.append("meta: collection is not a non-empty string")
if "index" in meta and (p := index_problem(meta["index"])):
problems.append(f"meta: {p}")
on_disk = {p.name: p for p in staging.iterdir()}
for name in sorted(set(on_disk) - set(described)):
problems.append(f"{name}: in the staging folder but not in the meta")
for name in sorted(set(described) - set(on_disk)):
problems.append(f"{name}: in the meta but not in the staging folder")
files: dict[str, bytes] = {}
entries = []
for name in sorted(set(on_disk) & set(described)):
if not NAME.match(name) or not on_disk[name].is_file() or on_disk[name].is_symlink():
problems.append(f"{name}: not a plain file named [a-z0-9._-]")
continue
problems += [f"{name}: {p}" for p in describe_problems(described[name])]
data = on_disk[name].read_bytes()
facts, why = file_facts(data)
if why:
problems.append(f"{name}: {why}")
continue
files[name] = data
entries.append({**described[name], **facts, "path": f"{KILDE}/{name}"})
if problems:
raise Refused(problems)
doc: dict[str, Any] = {"collection": meta["collection"], "frozen_at": when, "schema": SCHEMA,
"sources": entries, "subset_of": None}
if "index" in meta:
doc["index"] = meta["index"]
write_root(out, files, doc)
return doc
TEXT_READERS = (".md", ".markdown", ".txt")
READERS = TEXT_READERS + readers.TYPES
# Every line break str.splitlines() knows besides \n; a source keeps \n only, so line N is the same for every reader.
LINE_BREAKS = re.compile("\r\n|[\r\x0b\x0c\x1c\x1d\x1e\x85\u2028\u2029]")
FOLDER_KIND = "document"
ORIGINAL = "original" # beside kilde/: the PDF and Office files as they were, so the view can open them at the place
def safe_name(rel: str) -> str:
"""The kilde/ name of a file at rel (a path with "/") in a folder: see the folder command."""
parts = [re.sub(r"-+", "-", re.sub(r"[^a-z0-9._-]", "-", ascii_fold(part))).strip("-") or "x"
for part in rel.split("/")]
return "__".join(parts)
def normalised_text(data: bytes) -> tuple[bytes, list[str]]:
"""The bytes of a text file as a source (UTF-8, lines broken at \n only) and what was changed to get them."""
changed: list[str] = []
if data.startswith(b"\xef\xbb\xbf"):
data, changed = data[3:], ["bom"]
try:
text = data.decode("utf-8")
except UnicodeDecodeError:
try:
text = data.decode("cp1252")
except UnicodeDecodeError:
text = data.decode("latin-1")
changed.append("cp1252")
flat = LINE_BREAKS.sub("\n", text)
if flat != text:
changed.append("line-breaks")
return flat.encode("utf-8"), changed
def walk(folder: Path) -> tuple[list[tuple[str, Path]], list[str]]:
"""(every file under folder that has a reader, as (relative path in NFC, path), sorted on the relative path), and
one line per thing skipped with its reason."""
found: list[tuple[str, Path]] = []
skipped: list[str] = []
def visit(d: Path) -> None:
for p in sorted(d.iterdir()):
rel = unicodedata.normalize("NFC", p.relative_to(folder).as_posix())
if p.name.startswith("."):
skipped.append(f"{rel}: skipped, hidden")
elif p.is_symlink():
skipped.append(f"{rel}: skipped, a link")
elif p.is_dir():
visit(p)
elif not p.is_file():
skipped.append(f"{rel}: skipped, not a plain file")
elif p.suffix.lower() not in READERS:
skipped.append(f"{rel}: skipped, no reader for {p.suffix.lower() or 'a file without a suffix'}")
else:
found.append((rel, p))
visit(folder)
return sorted(found), sorted(skipped)
def freeze_folder(folder: Path, collection: str, out: Path,
frozen_at: str | None = None) -> tuple[dict[str, Any], list[str]]:
"""The folder command: (the manifest written, the lines of what was skipped)."""
when = checked_frozen_at(frozen_at)
if out.exists():
raise RunError(f"{out} exists; refusing to overwrite")
if not folder.is_dir():
raise RunError(f"{folder} is not a directory")
if not nonempty_str(collection):
raise RunError("the collection has no name")
found, skipped = walk(folder)
files: dict[str, bytes] = {}
entries: list[dict[str, Any]] = []
origin: dict[str, list[str]] = {}
missing: list[str] = []
originals: dict[str, bytes] = {}
for rel, path in found:
raw = path.read_bytes()
read: dict[str, Any] = {}
if path.suffix.lower() in readers.TYPES:
try:
rendering = readers.render(path)
except readers.NoReader as e:
missing.append(str(e))
continue
except readers.Unreadable as e:
skipped.append(f"{rel}: skipped, unreadable: {e}")
continue
if not rendering.text.strip():
skipped.append(f"{rel}: skipped, no text (a scanned page needs text recognition first)"
if path.suffix.lower() == ".pdf" else f"{rel}: skipped, no text")
continue
if rendering.left_out:
skipped.append(f"{rel}: {rendering.left_out} hidden parts left out")
data, name = rendering.text.encode("utf-8"), safe_name(rel) + ".md"
changed: list[str] = []
copy = f"{ORIGINAL}/{safe_name(rel)}"
originals[copy] = raw
read = {"left_out": rendering.left_out, "original_copy": copy, "original_sha256": sha256(raw),
"places": rendering.locators, "reader": path.suffix.lower()[1:]}
else:
data, changed = normalised_text(raw)
if not data.strip():
skipped.append(f"{rel}: skipped, empty")
continue
name = safe_name(rel)
if RELEASE.match(name): # a folder's document has one main topic, never a release note's name (B13)
name = name[:-len(".md")] + "-doc.md"
origin.setdefault(name, []).append(rel)
files[name] = data
facts, _ = file_facts(data)
lang = language(data.decode("utf-8"))
entries.append({"kind": FOLDER_KIND, "main_topic": True, "normalised": changed, "original": rel,
"url": path.resolve().as_uri(), **facts, **read, **({"lang": lang} if lang else {}),
"path": f"{KILDE}/{name}"})
problems = sorted(set(missing)) + [f"{name}: the safe name of {' and '.join(rels)}"
for name, rels in sorted(origin.items()) if len(rels) > 1]
if problems:
raise Refused(problems)
if not entries:
raise RunError(f"{folder} holds no document with a reader ({', '.join(READERS)})")
entries.sort(key=lambda e: str(e["path"]))
doc: dict[str, Any] = {"collection": collection, "folder": str(folder.resolve()), "frozen_at": when,
"schema": SCHEMA, "sources": entries, "subset_of": None}
write_root(out, files, doc, originals)
return doc, sorted(skipped)
def subset(root: Path, paths_file: Path, out: Path, frozen_at: str | None = None) -> dict[str, Any]:
when = checked_frozen_at(frozen_at)
if out.exists():
raise RunError(f"{out} exists; refusing to overwrite")
problems = check(root)
if problems:
raise RunError(f"{root} fails check: {'; '.join(problems)}")
text = paths_file.read_text(encoding="utf-8")
wanted = text.split("\n")
if wanted[-1] == "":
wanted.pop()
if not wanted:
raise RunError(f"{paths_file} lists no path")
if "" in wanted or wanted != sorted(set(wanted)):
raise RunError(f"{paths_file} is not one path per line, sorted, without duplicates or empty lines")
manifest_bytes = (root / MANIFEST).read_bytes()
doc: dict[str, Any] = json.loads(manifest_bytes)
by_path = {e["path"]: e for e in doc["sources"]}
unknown = [p for p in wanted if p not in by_path]
if unknown:
raise RunError(f"not in {root / MANIFEST}: {', '.join(unknown)}")
part = {**doc, "frozen_at": when, "sources": [by_path[p] for p in wanted],
"subset_of": {"frozen_at": doc["frozen_at"], "manifest_sha256": sha256(manifest_bytes)}}
write_root(out, {p[len(KILDE) + 1:]: (root / p).read_bytes() for p in wanted}, part,
{e["original_copy"]: (root / e["original_copy"]).read_bytes() for e in part["sources"]
if "original_copy" in e})
return part
def sound_manifest(root: Path) -> tuple[dict[str, Any], dict[str, str]]:
"""The manifest of a root that passes check, and its stamp (frozen_at and the sha256 of the manifest's bytes)."""
problems = check(root)
if problems:
raise RunError(f"{root} fails check: {'; '.join(problems)}")
data = (root / MANIFEST).read_bytes()
doc: dict[str, Any] = json.loads(data)
return doc, {"frozen_at": doc["frozen_at"], "manifest_sha256": sha256(data)}
def diff(old: Path, new: Path) -> dict[str, Any]:
"""K4 item 1: the sources of NEW that are new, changed or gone since OLD, each list sorted on path."""
old_doc, old_stamp = sound_manifest(old)
new_doc, new_stamp = sound_manifest(new)
if old_doc["collection"] != new_doc["collection"]:
raise RunError(f"{old} is collection {old_doc['collection']!r} and {new} is {new_doc['collection']!r}; "
"the paths of two collections can only match by chance")
before = {e["path"]: e for e in old_doc["sources"]}
after = {e["path"]: e for e in new_doc["sources"]}
changed: list[dict[str, Any]] = []
gone: list[dict[str, Any]] = []
new_sources: list[dict[str, Any]] = []
unchanged = 0
for path in sorted(before.keys() | after.keys()):
a, b = before.get(path), after.get(path)
if b is None:
assert a is not None
gone.append({"kind": a["kind"], **as_known(a), "path": path})
elif a is None:
new_sources.append({"kind": b["kind"], **as_known(b), "path": path})
elif a["sha256"] != b["sha256"] or a["kind"] != b["kind"]:
changed.append({"kind": b["kind"], "new_sha256": b["sha256"], "old_sha256": a["sha256"], **as_known(b),
"path": path})
else:
unchanged += 1
return {"changed": changed, "gone": gone, "new": new_stamp, "new_sources": new_sources, "old": old_stamp,
"schema": DIFF_SCHEMA, "unchanged": unchanged}
def as_known(entry: dict[str, Any]) -> dict[str, Any]:
"""The file as the user knows it ({"original": path in the folder}) for a source read from a folder, else {}."""
return {"original": entry["original"]} if isinstance(entry.get("original"), str) else {}
def has_change(report: dict[str, Any]) -> bool:
return bool(report["changed"] or report["gone"] or report["new_sources"])
def read_diff(path: Path) -> dict[str, str]:
"""The changed and gone paths of a diff file, by path: "changed" or "gone". RunError when it is not a K4 diff."""
doc = load_json(path)
if not isinstance(doc, dict) or doc.get("schema") != DIFF_SCHEMA:
raise RunError(f"{path} is not a {DIFF_SCHEMA} file")
out: dict[str, str] = {}
for change in ("changed", "gone"):
entries = doc.get(change)
if not isinstance(entries, list) or not all(isinstance(e, dict) and isinstance(e.get("path"), str)
for e in entries):
raise RunError(f"{path}: {change} is not a list of entries with a path")
out.update({e["path"]: change for e in entries})
return out
def page_resources(path: Path, rel: str) -> list[str]:
"""Every sources[].resource string in the frontmatter of one page; [] for a page without frontmatter."""
lines = path.read_text(encoding="utf-8").split("\n")
if lines[0].strip() != "---":
return []
end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
if end is None:
raise RunError(f"{rel}: the frontmatter is not closed with ---")
try:
data = yaml.safe_load("\n".join(lines[1:end]))
except yaml.YAMLError as exc:
raise RunError(f"{rel}: the frontmatter does not parse: {str(exc).splitlines()[0]}") from None
if data is None:
return []
if not isinstance(data, dict):
raise RunError(f"{rel}: the frontmatter is not a mapping")
cited = data.get("sources") or []
if not isinstance(cited, list):
raise RunError(f"{rel}: sources is not a list")
return [s["resource"] for s in cited if isinstance(s, dict) and isinstance(s.get("resource"), str)]
def marks(collection: Path, diff_path: Path) -> dict[str, Any]:
"""K4 item 2: the pages built on a changed or gone source, sorted on page, their sources sorted on path."""
changes = read_diff(diff_path)
if not collection.is_dir():
raise RunError(f"{collection} is not a directory")
pages: list[dict[str, Any]] = []
for p in sorted(collection.rglob("*.md")):
if not p.is_file():
continue
rel = p.relative_to(collection).as_posix()
hit = {r[1:] for r in page_resources(p, rel) if r.startswith("/") and r[1:] in changes}
if hit:
pages.append({"page": rel, "sources": [{"change": changes[s], "path": s} for s in sorted(hit)]})
pages.sort(key=lambda e: e["page"])
return {"pages": pages, "schema": MARKS_SCHEMA}
WATCH_SCHEMA = "okf-wiki/folder-watch/1"
WATCH_STATE = "watch-state.json"
LATEST = "latest"
def source_folder(doc: dict[str, Any]) -> Path:
"""The folder a copy was read from: its `folder`, or for a copy frozen before that field, the one folder its
entries name (each `url` less its `original` path). RunError when the copy was not read from a folder."""
if nonempty_str(doc.get("folder")):
return Path(doc["folder"])
found: set[Path] = set()
for e in doc["sources"]:
url, original = e.get("url"), e.get("original")
if not (isinstance(url, str) and url.startswith("file://") and nonempty_str(original)):
continue
path = Path(urllib.parse.unquote(urllib.parse.urlparse(url).path))
depth = len(Path(original).parts)
if len(path.parts) <= depth:
continue
found.add(Path(*path.parts[:-depth]))
if not found:
raise RunError("the copy was not read from a folder (no folder, and no entry with a file:// url and an "
"original path); only a folder copy can be read again")
if len(found) > 1:
raise RunError(f"the entries name more than one folder ({', '.join(sorted(map(str, found)))}); "
"read the folder again with the folder command")
return found.pop()
def sound_root(root: Path) -> dict[str, Any]:
problems = check(root)
if problems:
raise RunError(f"{root} fails check: {'; '.join(problems[:5])}")
return load_manifest(root)
def reread(root: Path, out: Path, frozen_at: str | None = None) -> tuple[dict[str, Any], list[str], Path]:
"""The reread command: the folder ROOT was read from, read again as the same collection into OUT."""
doc = sound_root(root)
folder = source_folder(doc)
if not folder.is_dir():
raise RunError(f"the folder {folder} that {root} was read from is not there")
try:
new, skipped = freeze_folder(folder, doc["collection"], out, frozen_at)
except Refused as exc:
raise RunError(f"{folder} cannot be read again: {'; '.join(exc.problems)}") from exc
return new, skipped, folder
def watch_lines(report: dict[str, Any]) -> list[str]:
"""One line per difference, "change path [new sha256]", so a later change of the same file is a new line."""
lines = [f"new {e['path']}" for e in report["new_sources"]]
lines += [f"changed {e['path']} {e['new_sha256']}" for e in report["changed"]]
lines += [f"gone {e['path']}" for e in report["gone"]]
return sorted(lines)
def watch(root: Path, into: Path, bundle: Path | None, frozen_at: str | None = None) -> tuple[str, bool]:
"""The watch command: (the report, whether something is new since the last run). See the module doc."""
doc = sound_root(root)
folder = source_folder(doc).resolve()
target = into.resolve()
if target == folder or folder in target.parents:
raise RunError(f"{into} is inside the folder {folder}; the next reading would take the watch's files for "
"documents. Put it beside the folder")
state_path = into / WATCH_STATE
state = load_json(state_path) if state_path.is_file() else None
if state is not None and not (isinstance(state, dict) and state.get("schema") == WATCH_SCHEMA
and isinstance(state.get("entries"), list)):
raise RunError(f"{state_path} is not a watch state ({WATCH_SCHEMA})")
into.mkdir(parents=True, exist_ok=True)
work = Path(tempfile.mkdtemp(prefix=".watch-", dir=into))
try:
_, skipped, _ = reread(root, work / LATEST, frozen_at)
report = diff(root, work / LATEST)
(work / LATEST / "diff.json").write_text(dump(report), encoding="utf-8")
marked: dict[str, int] = {}
pages = 0
if bundle is not None:
found = marks(bundle, work / LATEST / "diff.json")
(work / LATEST / "marks.json").write_text(dump(found), encoding="utf-8")
pages = len(found["pages"])
for pg in found["pages"]:
for src in pg["sources"]:
marked[src["path"]] = marked.get(src["path"], 0) + 1
latest = into / LATEST
if latest.exists():
latest.rename(work / "replaced")
(work / LATEST).rename(latest)
finally:
shutil.rmtree(work, ignore_errors=True)
entries = watch_lines(report)
same_copy = state is not None and state.get("frozen") == report["old"]
seen = {str(e) for e in state["entries"]} if state is not None and same_copy else set()
news = [e for e in entries if e not in seen]
resolved = sorted(seen - set(entries))
state_path.write_text(dump({"entries": entries, "frozen": report["old"], "schema": WATCH_SCHEMA}),
encoding="utf-8")
by_path = {e["path"]: e for k in ("new_sources", "changed", "gone") for e in report[k]}
counts = (f"{len(report['new_sources'])} new, {len(report['changed'])} changed, {len(report['gone'])} gone, "
f"{report['unchanged']} unchanged")
lines = [f"watch: {folder} read again into {latest}, against {root} (frozen {report['old']['frozen_at']}): "
f"{counts}"]
if not news:
lines.append(" nothing differs" if not entries else " nothing new since the last run")
else:
lines.append("New since the last run:" if seen or same_copy else "Every difference (first run against this copy):")
for e in news:
change, path = e.split(" ")[:2]
known = by_path[path].get("original", path)
n = marked.get(path)
lines.append(f" {change:<8} {known} ({path})" + (f", {n} page(s) marked" if n else ""))
if resolved:
lines.append("No longer differs from the frozen copy:")
lines += [f" {e.split(' ')[1]}" for e in resolved]
if skipped:
lines.append("Skipped in the folder:")
lines += [f" {line}" for line in skipped]
if bundle is not None:
lines.append(f"{pages} page(s) marked in {bundle}; the list: {latest / 'marks.json'}")
lines.append(f"Every difference: {latest / 'diff.json'}. This only warns; rebuild from {latest} to update.")
return "\n".join(lines) + "\n", bool(news)
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
c = sub.add_parser("check")
c.add_argument("root")
f = sub.add_parser("freeze")
f.add_argument("staging")
f.add_argument("--meta", required=True)
f.add_argument("--out", required=True)
f.add_argument("--frozen-at")
fo = sub.add_parser("folder")
fo.add_argument("folder")
fo.add_argument("--collection", required=True)
fo.add_argument("--out", required=True)
fo.add_argument("--frozen-at")
s = sub.add_parser("subset")
s.add_argument("root")
s.add_argument("--paths", required=True)
s.add_argument("--out", required=True)
s.add_argument("--frozen-at")
d = sub.add_parser("diff")
d.add_argument("old")
d.add_argument("new")
m = sub.add_parser("marks")
m.add_argument("collection")
m.add_argument("--diff", required=True)
r = sub.add_parser("reread")
r.add_argument("root")
r.add_argument("--out", required=True)
r.add_argument("--frozen-at")
w = sub.add_parser("watch")
w.add_argument("root")
w.add_argument("--into", required=True)
w.add_argument("--bundle")
w.add_argument("--frozen-at")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
if args.command == "check":
root = Path(args.root)
problems = check(root)
n = len(load_manifest(root)["sources"])
if problems:
print(f"check: {len(problems)} problems in {root} ({n} sources listed)")
for p in problems:
print(f" {p}")
return 1
print(f"check: {n} sources in {root}, every file present and listed")
return 0
if args.command == "diff":
report = diff(Path(args.old), Path(args.new))
sys.stdout.write(dump(report))
return 1 if has_change(report) else 0
if args.command == "marks":
marked = marks(Path(args.collection), Path(args.diff))
sys.stdout.write(dump(marked))
return 1 if marked["pages"] else 0
if args.command == "reread":
new, skipped, folder = reread(Path(args.root), Path(args.out), args.frozen_at)
print(f"reread: {len(new['sources'])} sources from {folder} -> {args.out}; {len(skipped)} skipped")
for line in skipped:
print(f" {line}")
return 0
if args.command == "watch":
text, news = watch(Path(args.root), Path(args.into), Path(args.bundle) if args.bundle else None,
args.frozen_at)
sys.stdout.write(text)
return 1 if news else 0
if args.command == "folder":
doc, skipped = freeze_folder(Path(args.folder), args.collection, Path(args.out), args.frozen_at)
print(f"folder: {len(doc['sources'])} sources -> {args.out}; {len(skipped)} skipped")
for line in skipped:
print(f" {line}")
return 0
if args.command == "freeze":
doc = freeze(Path(args.staging), Path(args.meta), Path(args.out), args.frozen_at)
else:
doc = subset(Path(args.root), Path(args.paths), Path(args.out), args.frozen_at)
print(f"{args.command}: {len(doc['sources'])} sources -> {args.out}")
return 0
except Refused as exc:
print(f"{args.command} refused: {len(exc.problems)} problems, nothing written", file=sys.stderr)
for p in exc.problems:
print(f" {p}", file=sys.stderr)
return 1
except (RunError, OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
print(f"{args.command} cannot run: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())

31
okfwiki/transcript.py Normal file
View file

@ -0,0 +1,31 @@
"""Reads Claude Code session transcripts (.jsonl), as written under ~/.claude/projects/<slug>/. No model call."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
USAGE_FIELDS = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens", "output_tokens")
def is_agent_type(recorded: object, name: str) -> bool:
"""Whether a subagent's recorded agentType is the agent `name`, started bare ("okf-wiki-reader") or from a
plugin, which Claude Code records with the plugin's namespace ("<plugin>:okf-wiki-reader")."""
if not isinstance(recorded, str):
return False
plugin, sep, bare = recorded.rpartition(":")
return bare == name and (not sep or bool(plugin))
def read_transcript(path: Path) -> list[dict[str, Any]]:
"""Every JSON object line of a transcript; a line that does not parse is skipped."""
lines = []
for raw in path.read_text(encoding="utf-8").splitlines():
try:
line = json.loads(raw)
except json.JSONDecodeError:
continue
if isinstance(line, dict):
lines.append(line)
return lines

1092
okfwiki/view.py Normal file

File diff suppressed because it is too large Load diff