okf-wiki 1.1.1
First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
commit
0fd86a23a3
227 changed files with 96077 additions and 0 deletions
1
okfwiki/__init__.py
Normal file
1
okfwiki/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
"""okf-wiki: tools that check and render an OKF wiki. No tool here calls a language model (CLAUDE.md rule 1)."""
|
||||
969
okfwiki/audit.py
Normal file
969
okfwiki/audit.py
Normal file
|
|
@ -0,0 +1,969 @@
|
|||
"""The auditor: checks an OKF wiki bundle against its raw sources. It never calls a language model (CLAUDE.md rule 1).
|
||||
|
||||
Usage:
|
||||
python3 -m okfwiki.audit BUNDLE SOURCES
|
||||
BUNDLE is the wiki directory. SOURCES is the raw-source root that a bundle-absolute `sources[].resource`
|
||||
(`/kilde/x.md`) resolves against, and that holds the source manifest `source-manifest.json` (contract K1). L1a reads
|
||||
raw sources as UTF-8 text with a `{ line: N }` locator only.
|
||||
|
||||
Output on stdout, one JSON object: {"findings": [{"check", "group", "file", "line", "message"}, ...],
|
||||
"summary": {<group>: n}, "link_rule": {"line", "main_pages": {"pass", "of"}, "linked": {"pass", "of"}, "misplaced",
|
||||
"broken", "not_to_concept", "max_out_degree", "provenance"}}. `link_rule` is criterion 6 as numbers (contract K2
|
||||
point 7): "main pages: N of N sources · linked from a main page: N of N pages".
|
||||
Exit 0 clean, 1 at least one finding, 2 the audit could not run (bad arguments; no valid K1 file at SOURCES, since the
|
||||
main-page rule cannot run without it, K2 point 6; or no main-topic source in K1 and no concept page, no denominator).
|
||||
|
||||
Four groups are reported apart, because they answer different questions:
|
||||
- spec-11: OKF v0.2 SPEC section 11 conformance (PLAN.md criterion 5).
|
||||
- links: okf-wiki's own link rules. SPEC 6.1 says consumers MUST tolerate broken links, so these are not section 11.
|
||||
- citations: every body sentence carries a footnote, every footnote a source, every source a verbatim quote at its
|
||||
locator (PLAN.md criterion 1, format choice f).
|
||||
- format: format choices (a), (b), (c) and (e) in PLAN.md section Rammer.
|
||||
The exact definitions of the link classes, including link-misplaced (b) of criterion 6 and the main-page rule
|
||||
(contract K2: main-page-missing, main-page-duplicate, main-page-invalid, no-main-page-link), are in
|
||||
gates/mutants.manifest.json under provenance.link_rules. The rule of 04.10.2026 (an inbound link from any other concept
|
||||
page) is no finding any more; link_rule_report() still counts it, for the linker and the trial.
|
||||
|
||||
The function audit() reports the main-page rule only when SOURCES holds a K1 file (K2 point 18); without one it gives
|
||||
the other checks, as the trial's tests and the assembler's call it on roots without K1.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Callable
|
||||
from urllib.parse import unquote
|
||||
|
||||
import yaml
|
||||
|
||||
from okfwiki.common import place_fits
|
||||
|
||||
RESERVED = ("index.md", "log.md") # SPEC 3.1
|
||||
GROUPS = ("spec-11", "links", "citations", "format")
|
||||
GROUP_OF = {
|
||||
"spec11-1-frontmatter": "spec-11",
|
||||
"spec11-2-type": "spec-11",
|
||||
"spec11-3-index": "spec-11",
|
||||
"spec11-3-log": "spec-11",
|
||||
"broken-link": "links",
|
||||
"orphan-page": "links",
|
||||
"index-coverage": "links",
|
||||
"link-misplaced": "links",
|
||||
"main-page-missing": "links",
|
||||
"main-page-duplicate": "links",
|
||||
"main-page-invalid": "links",
|
||||
"no-main-page-link": "links",
|
||||
"uncited-sentence": "citations",
|
||||
"quote-not-verbatim": "citations",
|
||||
"quote-wrong-locator": "citations",
|
||||
"quote-wrong-place": "citations",
|
||||
"quote-missing": "citations",
|
||||
"locator-unsupported": "citations",
|
||||
"source-unreadable": "citations",
|
||||
"source-missing-id": "citations",
|
||||
"source-duplicate-id": "citations",
|
||||
"footnote-unknown-id": "citations",
|
||||
"resource-not-bundle-absolute": "format",
|
||||
"body-link-bundle-absolute": "format",
|
||||
"timestamp-form": "format",
|
||||
"verified-not-human": "format",
|
||||
}
|
||||
|
||||
|
||||
class _Loader(yaml.SafeLoader):
|
||||
"""SafeLoader without the timestamp resolver, as Google's reference `document.py` does (format choice c)."""
|
||||
|
||||
|
||||
_Loader.yaml_implicit_resolvers = {
|
||||
ch: [(tag, rx) for tag, rx in resolvers if tag != "tag:yaml.org,2002:timestamp"]
|
||||
for ch, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items()
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Finding:
|
||||
check: str
|
||||
group: str
|
||||
file: str
|
||||
line: int | None
|
||||
message: str
|
||||
|
||||
|
||||
def finding(check: str, file: str, line: int | None, message: str) -> Finding:
|
||||
return Finding(check=check, group=GROUP_OF[check], file=file, line=line, message=message)
|
||||
|
||||
|
||||
# --- documents -------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Document:
|
||||
"""One markdown file split as SPEC section 4 says: a frontmatter block, then the body."""
|
||||
|
||||
rel: str
|
||||
has_frontmatter: bool
|
||||
frontmatter: dict[str, Any] | None # None when absent or not parseable to a mapping
|
||||
frontmatter_error: str
|
||||
body: str
|
||||
body_start: int # 1-based file line of the body's first line
|
||||
|
||||
|
||||
def read_document(path: Path, rel: str) -> Document:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
lines = text.split("\n")
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return Document(rel, False, None, "", text, 1)
|
||||
end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
|
||||
if end is None:
|
||||
return Document(rel, True, None, "the frontmatter block is not closed with ---", "", len(lines) + 1)
|
||||
body = "\n".join(lines[end + 1:])
|
||||
try:
|
||||
data = yaml.load("\n".join(lines[1:end]), Loader=_Loader)
|
||||
except yaml.YAMLError as exc:
|
||||
first = str(exc).splitlines()[0] if str(exc) else "YAML error"
|
||||
return Document(rel, True, None, f"the frontmatter does not parse: {first}", body, end + 2)
|
||||
if data is None:
|
||||
data = {}
|
||||
if not isinstance(data, dict):
|
||||
return Document(rel, True, None, "the frontmatter is not a YAML mapping", body, end + 2)
|
||||
return Document(rel, True, data, "", body, end + 2)
|
||||
|
||||
|
||||
# --- markdown body ---------------------------------------------------------------------------------------------
|
||||
|
||||
# A code span: a backtick run not escaped and not part of a longer run, closed by a run of the same length (6.1).
|
||||
CODE_SPAN = re.compile(r"(?<![`\\])(`+)(?!`)(.+?)(?<!`)\1(?!`)", re.S)
|
||||
FENCE = re.compile(r"^(?:[ \t]*>)*[ \t]*(`{3,}|~{3,})") # in a list item or a quote block too
|
||||
LINK = re.compile(r"(?<![!\\])\[([^\]\n]*)\]\(\s*<?([^()\s<>]*)>?(?:\s+\"[^\"]*\")?\s*\)")
|
||||
FOOTNOTE_REF = re.compile(r"\[\^([^\]\s]+)\](?!:)")
|
||||
FOOTNOTE_DEF = re.compile(r"^\[\^([^\]\s]+)\]:")
|
||||
HEADING = re.compile(r"^ {0,3}(#{1,6})(?:\s+(.*?))?\s*#*\s*$")
|
||||
LIST_ITEM = re.compile(r"^\s*(?:[-*+]|\d+[.)])\s+")
|
||||
SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:")
|
||||
# A sentence ends at . ! or ?, optionally followed by closing quotes or brackets and by footnote references,
|
||||
# then whitespace. sentences() skips an end followed by a lowercase letter, which keeps abbreviations such as
|
||||
# "e.g. overnight" inside their sentence.
|
||||
SENTENCE_END = re.compile(r"[.!?][\"'»”’)\]]*(?:\[\^[^\]\s]+\])*(?=\s+(\S))")
|
||||
# A period after one of these ends no sentence, whatever follows ("kl. 14", "f.eks. Oslo", "e.g. Claude Code"). Only
|
||||
# abbreviations that seldom end a sentence are here: "osv.", "etc." and "m.m." usually close one and still cut. A
|
||||
# single capital letter before the period is an initial and does not cut either.
|
||||
ABBREVIATIONS = frozenset(
|
||||
"f.eks bl.a kl ca nr jf jfr dvs evt ev inkl ekskl mht pga ift iht vedr ang tlf stk pkt kap avd ref hhv mill mrd"
|
||||
" e.g i.e cf vs approx incl mr mrs ms dr prof".split())
|
||||
ABBREVIATION_BEFORE = re.compile(r"([^\s(\[\"'«]+)\.$")
|
||||
LINK_ONLY = re.compile(r"^(?:\s*\[[^\]\n]*\]\([^()\s]*\)\s*[,;]?)+\s*$")
|
||||
|
||||
|
||||
def fence_marker(raw: str) -> re.Match[str] | None:
|
||||
"""The fence run that opens or closes a fenced block on this line (CommonMark 4.5), or None. A backtick run with a
|
||||
backtick after it is an inline code span, not a fence."""
|
||||
f = FENCE.match(raw)
|
||||
if f and f.group(1)[0] == "`" and "`" in raw[f.end():]:
|
||||
return None
|
||||
return f
|
||||
|
||||
|
||||
def closes_fence(f: re.Match[str] | None, raw: str, fence: str) -> bool:
|
||||
"""A fence closes on a run of the opening character at least as long as the opening run, with nothing after it."""
|
||||
return bool(f and f.group(1)[0] == fence[0] and len(f.group(1)) >= len(fence) and not raw[f.end():].strip())
|
||||
|
||||
|
||||
def mask_code(text: str) -> str:
|
||||
"""Blank out code so nothing inside it reads as a link, footnote or sentence end; keeps offsets and line breaks.
|
||||
|
||||
A fenced block (backticks or tildes) is masked line by line after its opening run. An inline code span pairs
|
||||
backtick runs of equal length only inside one block: a blank line, a heading, a list item or a footnote
|
||||
definition starts a new one, and a backslash before a backtick makes it literal (CommonMark 6.1)."""
|
||||
out: list[str] = []
|
||||
block: list[str] = []
|
||||
|
||||
def flush() -> None:
|
||||
if block:
|
||||
out.append(CODE_SPAN.sub(lambda m: m.group(1) + "x" * len(m.group(2)) + m.group(1), "\n".join(block)))
|
||||
block.clear()
|
||||
|
||||
fence = ""
|
||||
for raw in text.split("\n"):
|
||||
f = fence_marker(raw)
|
||||
if fence:
|
||||
if closes_fence(f, raw, fence):
|
||||
fence = ""
|
||||
out.append(raw)
|
||||
else:
|
||||
out.append("x" * len(raw))
|
||||
elif f:
|
||||
flush()
|
||||
fence = f.group(1)
|
||||
out.append(raw[:f.end()] + "x" * len(raw[f.end():]))
|
||||
elif not raw.strip() or HEADING.match(raw):
|
||||
flush()
|
||||
out.append(CODE_SPAN.sub(lambda m: m.group(1) + "x" * len(m.group(2)) + m.group(1), raw))
|
||||
else:
|
||||
if LIST_ITEM.match(raw) or FOOTNOTE_DEF.match(raw.strip()):
|
||||
flush()
|
||||
block.append(raw)
|
||||
flush()
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def link_matches(text: str, masked: str | None = None) -> list[re.Match[str]]:
|
||||
"""The inline links of text. A [ inside a code span opens no link (CommonMark 6.1): a match that opens in code
|
||||
and closes outside it is passed over from its bracket, so the link after it is found. A link written wholly
|
||||
inside code is kept, and placed_links calls it code. masked is mask_code(text), or the same slice of it."""
|
||||
masked = mask_code(text) if masked is None else masked
|
||||
out: list[re.Match[str]] = []
|
||||
pos = 0
|
||||
while (m := LINK.search(text, pos)) is not None:
|
||||
if masked[m.start()] != text[m.start()] and masked[m.end() - 1] == text[m.end() - 1]:
|
||||
pos = m.start() + 1
|
||||
continue
|
||||
out.append(m)
|
||||
pos = m.end()
|
||||
return out
|
||||
|
||||
|
||||
def sub_links(repl: Callable[[re.Match[str]], str], text: str) -> str:
|
||||
"""LINK.sub over link_matches: a bracket inside a code span is left as it stands."""
|
||||
parts: list[str] = []
|
||||
last = 0
|
||||
for m in link_matches(text):
|
||||
parts += [text[last:m.start()], repl(m)]
|
||||
last = m.end()
|
||||
return "".join(parts) + text[last:]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Block:
|
||||
line: int
|
||||
text: str
|
||||
is_list_item: bool
|
||||
|
||||
|
||||
def prose_blocks(doc: Document) -> list[Block]:
|
||||
"""Paragraphs and list items of the body; headings and footnote definitions are not prose."""
|
||||
blocks: list[Block] = []
|
||||
current: Block | None = None
|
||||
for i, raw in enumerate(doc.body.split("\n")):
|
||||
line_no = doc.body_start + i
|
||||
if not raw.strip():
|
||||
current = None
|
||||
continue
|
||||
if HEADING.match(raw) or FOOTNOTE_DEF.match(raw.strip()):
|
||||
current = None
|
||||
continue
|
||||
if LIST_ITEM.match(raw):
|
||||
current = Block(line_no, LIST_ITEM.sub("", raw, count=1), True)
|
||||
blocks.append(current)
|
||||
elif current is None:
|
||||
current = Block(line_no, raw.strip(), False)
|
||||
blocks.append(current)
|
||||
else:
|
||||
current.text += " " + raw.strip()
|
||||
return blocks
|
||||
|
||||
|
||||
def sentence_spans(text: str) -> list[tuple[int, int]]:
|
||||
"""(start, end) of every sentence in text, found on the code-masked text, so the offsets fit the original too.
|
||||
|
||||
A bundle link never moves a cut: the ends are found on the masked text with every bundle link outside code
|
||||
reduced to its anchor text (klink.reduce_links' notion), and each cut is mapped back to the linked text, after
|
||||
the whole link when it falls at the anchor's end. So "U.S. [primary](p.md) was" stays one sentence, as "U.S.
|
||||
primary was" is. Text without a bundle link is split exactly as before: the reduced text is the masked text."""
|
||||
masked = mask_code(text)
|
||||
plain, after = masked, list(range(1, len(masked) + 1)) # after[i]: offset in masked just after plain[i]
|
||||
links = [m for m in link_matches(text, masked) if masked[m.start()] == "[" and is_bundle_link(m.group(2))]
|
||||
if links:
|
||||
parts: list[str] = []
|
||||
after = []
|
||||
last = 0
|
||||
for m in links:
|
||||
parts += [masked[last:m.start()], masked[m.start(1):m.end(1)]]
|
||||
after += list(range(last + 1, m.start() + 1)) + list(range(m.start(1) + 1, m.end(1) + 1))
|
||||
if m.end(1) > m.start(1):
|
||||
after[-1] = m.end()
|
||||
last = m.end()
|
||||
parts.append(masked[last:])
|
||||
after += list(range(last + 1, len(masked) + 1))
|
||||
plain = "".join(parts)
|
||||
cuts: list[tuple[int, int]] = []
|
||||
start = 0
|
||||
for e in sentence_ends(plain):
|
||||
end = after[e - 1]
|
||||
cuts.append((start, end))
|
||||
start = end
|
||||
cuts.append((start, len(masked)))
|
||||
out: list[tuple[int, int]] = []
|
||||
for a, b in cuts:
|
||||
piece = masked[a:b]
|
||||
if not re.search(r"\w", FOOTNOTE_REF.sub("", piece)):
|
||||
continue
|
||||
lead = len(piece) - len(piece.lstrip())
|
||||
out.append((a + lead, a + len(piece.rstrip())))
|
||||
return out
|
||||
|
||||
|
||||
def sentence_ends(masked: str) -> list[int]:
|
||||
"""The offset just after each sentence end of a code-masked text."""
|
||||
ends: list[int] = []
|
||||
for m in SENTENCE_END.finditer(masked):
|
||||
if m.group(1).islower():
|
||||
continue
|
||||
word = ABBREVIATION_BEFORE.search(masked, max(0, m.start() - 20), m.start() + 1)
|
||||
w = word.group(1) if word else ""
|
||||
if masked[m.start()] == "." and (w.lower() in ABBREVIATIONS and w in (w.lower(), w.capitalize())
|
||||
or len(w) == 1 and w.isupper()):
|
||||
continue # an abbreviation, or an initial ("Steven L. Rickman")
|
||||
ends.append(m.end())
|
||||
return ends
|
||||
|
||||
|
||||
def sentences(text: str) -> list[str]:
|
||||
masked = mask_code(text)
|
||||
return [masked[a:b] for a, b in sentence_spans(text)]
|
||||
|
||||
|
||||
def body_links(doc: Document) -> list[tuple[int, str]]:
|
||||
"""(file line, target) for every inline markdown link in the body, outside code spans."""
|
||||
out: list[tuple[int, str]] = []
|
||||
for i, raw in masked_lines(doc.body):
|
||||
for m in LINK.finditer(raw):
|
||||
out.append((doc.body_start + i, m.group(2)))
|
||||
return out
|
||||
|
||||
|
||||
def masked_lines(body: str) -> list[tuple[int, str]]:
|
||||
"""(index, masked line) per line of the unmasked body. mask_code also masks a fenced block's line breaks, so
|
||||
the masked text is cut at the body's own line offsets: a line after a fenced block keeps its own number."""
|
||||
masked = mask_code(body)
|
||||
out: list[tuple[int, str]] = []
|
||||
offset = 0
|
||||
for i, raw in enumerate(body.split("\n")):
|
||||
out.append((i, masked[offset:offset + len(raw)]))
|
||||
offset += len(raw) + 1
|
||||
return out
|
||||
|
||||
|
||||
def is_bundle_link(target: str) -> bool:
|
||||
return bool(target) and not target.startswith("#") and not SCHEME.match(target)
|
||||
|
||||
|
||||
def resolve(rel_file: str, target: str) -> str | None:
|
||||
"""Bundle-relative POSIX path a link or path names, or None if it leaves the bundle (SPEC 6.1, 6.2)."""
|
||||
path = unquote(target.split("#", 1)[0].split("?", 1)[0])
|
||||
if path.startswith("/"):
|
||||
joined = path.lstrip("/")
|
||||
else:
|
||||
joined = str(PurePosixPath(rel_file).parent / path)
|
||||
norm = os.path.normpath(joined).replace(os.sep, "/")
|
||||
if norm == ".":
|
||||
return ""
|
||||
if norm == ".." or norm.startswith("../"):
|
||||
return None
|
||||
return norm
|
||||
|
||||
|
||||
# --- the audit -------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Bundle:
|
||||
root: Path
|
||||
sources: Path
|
||||
docs: dict[str, Document] = field(default_factory=dict)
|
||||
_places: dict[str, list[Any]] | None = None
|
||||
|
||||
def places(self, rel: str) -> list[Any] | None:
|
||||
"""The place in the original of each line of the raw source at rel (K1 `places`, PDF and Office), or None."""
|
||||
if self._places is None:
|
||||
self._places = {}
|
||||
try:
|
||||
doc = json.loads((self.sources / "source-manifest.json").read_text(encoding="utf-8"))
|
||||
for e in doc.get("sources", []) if isinstance(doc, dict) else []:
|
||||
if isinstance(e, dict) and isinstance(e.get("path"), str) and isinstance(e.get("places"), list):
|
||||
self._places[e["path"]] = e["places"]
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return self._places.get(rel)
|
||||
|
||||
@property
|
||||
def concepts(self) -> list[Document]:
|
||||
return [d for rel, d in sorted(self.docs.items()) if PurePosixPath(rel).name not in RESERVED]
|
||||
|
||||
def named(self, name: str) -> list[Document]:
|
||||
return [d for rel, d in sorted(self.docs.items()) if PurePosixPath(rel).name == name]
|
||||
|
||||
def is_file(self, rel: str | None) -> bool:
|
||||
return rel is not None and rel != "" and (self.root / rel).is_file()
|
||||
|
||||
|
||||
def load_bundle(root: Path, sources: Path) -> Bundle:
|
||||
b = Bundle(root, sources)
|
||||
for p in sorted(root.rglob("*.md")):
|
||||
if p.is_file():
|
||||
rel = p.relative_to(root).as_posix()
|
||||
b.docs[rel] = read_document(p, rel)
|
||||
return b
|
||||
|
||||
|
||||
def check_spec11_concepts(b: Bundle) -> list[Finding]:
|
||||
"""SPEC 11.1 and 11.2 on every non-reserved .md file in the tree."""
|
||||
out: list[Finding] = []
|
||||
for d in b.concepts:
|
||||
if not d.has_frontmatter:
|
||||
out.append(finding("spec11-1-frontmatter", d.rel, 1, "no YAML frontmatter block (SPEC 11.1)"))
|
||||
elif d.frontmatter is None:
|
||||
out.append(finding("spec11-1-frontmatter", d.rel, 1, d.frontmatter_error + " (SPEC 11.1)"))
|
||||
else:
|
||||
t = d.frontmatter.get("type")
|
||||
if not isinstance(t, str) or not t.strip():
|
||||
out.append(finding("spec11-2-type", d.rel, 1, "no non-empty type field (SPEC 11.2)"))
|
||||
return out
|
||||
|
||||
|
||||
def body_headings(doc: Document) -> list[tuple[int, int, str]]:
|
||||
"""(file line, level, text) for every ATX heading in the body."""
|
||||
out: list[tuple[int, int, str]] = []
|
||||
for i, raw in enumerate(doc.body.split("\n")):
|
||||
m = HEADING.match(raw)
|
||||
if m:
|
||||
out.append((doc.body_start + i, len(m.group(1)), (m.group(2) or "").strip()))
|
||||
return out
|
||||
|
||||
|
||||
def check_spec11_index(b: Bundle) -> list[Finding]:
|
||||
"""SPEC 8 via 11.3: no frontmatter except `okf_version` at the bundle root; entries grouped under headings."""
|
||||
out: list[Finding] = []
|
||||
for d in b.named("index.md"):
|
||||
if d.has_frontmatter:
|
||||
at_root = d.rel == "index.md"
|
||||
if d.frontmatter is None:
|
||||
out.append(finding("spec11-3-index", d.rel, 1, d.frontmatter_error + " (SPEC 8)"))
|
||||
elif not at_root or set(d.frontmatter) - {"okf_version"}:
|
||||
out.append(finding("spec11-3-index", d.rel, 1,
|
||||
"an index file carries no frontmatter, except okf_version in the bundle-root index (SPEC 8)"))
|
||||
body_lines = d.body.split("\n")
|
||||
headings = body_headings(d)
|
||||
first_heading = headings[0][0] if headings else None
|
||||
before = [d.body_start + i for i, raw in enumerate(body_lines)
|
||||
if raw.strip() and (first_heading is None or d.body_start + i < first_heading)]
|
||||
if not headings:
|
||||
out.append(finding("spec11-3-index", d.rel, d.body_start, "no section heading; entries are grouped under headings (SPEC 8)"))
|
||||
elif before:
|
||||
out.append(finding("spec11-3-index", d.rel, before[0], "content before the first section heading (SPEC 8)"))
|
||||
return out
|
||||
|
||||
|
||||
DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
||||
|
||||
|
||||
def check_spec11_log(b: Bundle) -> list[Finding]:
|
||||
"""SPEC 9 via 11.3: date headings in YYYY-MM-DD form, entries under a date, newest first."""
|
||||
out: list[Finding] = []
|
||||
for d in b.named("log.md"):
|
||||
dates: list[tuple[int, date]] = []
|
||||
first_date_line: int | None = None
|
||||
for line, level, text in body_headings(d):
|
||||
if level == 1:
|
||||
continue
|
||||
if not DATE.match(text):
|
||||
out.append(finding("spec11-3-log", d.rel, line, f"date heading {text!r} is not ISO 8601 YYYY-MM-DD (SPEC 9)"))
|
||||
continue
|
||||
try:
|
||||
dates.append((line, date.fromisoformat(text)))
|
||||
except ValueError:
|
||||
out.append(finding("spec11-3-log", d.rel, line, f"date heading {text!r} is not a calendar date (SPEC 9)"))
|
||||
continue
|
||||
if first_date_line is None:
|
||||
first_date_line = line
|
||||
for (_, newer), (line, older) in zip(dates, dates[1:]):
|
||||
if older > newer:
|
||||
out.append(finding("spec11-3-log", d.rel, line, "entries are not newest first (SPEC 9)"))
|
||||
for i, raw in enumerate(d.body.split("\n")):
|
||||
line = d.body_start + i
|
||||
if LIST_ITEM.match(raw) and (first_date_line is None or line < first_date_line):
|
||||
out.append(finding("spec11-3-log", d.rel, line, "an entry that is not under a date heading (SPEC 9)"))
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def check_links(b: Bundle) -> list[Finding]:
|
||||
"""broken-link, orphan-page and index-coverage, as defined in the manifest's provenance.link_rules."""
|
||||
out: list[Finding] = []
|
||||
incoming: dict[str, set[str]] = {d.rel: set() for d in b.concepts}
|
||||
for d in b.concepts:
|
||||
for line, target in body_links(d):
|
||||
if not is_bundle_link(target):
|
||||
continue
|
||||
rel = resolve(d.rel, target)
|
||||
if not b.is_file(rel):
|
||||
out.append(finding("broken-link", d.rel, line, f"link target {target!r} is not a file in the bundle"))
|
||||
elif rel in incoming and rel != d.rel:
|
||||
incoming[rel].add(d.rel)
|
||||
for idx in b.named("index.md"):
|
||||
listed: set[str] = set()
|
||||
for line, target in body_links(idx):
|
||||
if not is_bundle_link(target):
|
||||
continue
|
||||
rel = resolve(idx.rel, target)
|
||||
if rel is not None and rel != "" and (b.root / rel).is_dir():
|
||||
continue
|
||||
if not b.is_file(rel):
|
||||
out.append(finding("index-coverage", idx.rel, line, f"index entry {target!r} is not a file"))
|
||||
continue
|
||||
assert rel is not None
|
||||
listed.add(rel)
|
||||
if rel in incoming:
|
||||
incoming[rel].add(idx.rel)
|
||||
directory = PurePosixPath(idx.rel).parent
|
||||
for d in b.concepts:
|
||||
if PurePosixPath(d.rel).parent == directory and d.rel not in listed:
|
||||
out.append(finding("index-coverage", idx.rel, None, f"{d.rel} is in the directory but not in its index (SPEC 8)"))
|
||||
for rel, sources in sorted(incoming.items()):
|
||||
if not sources:
|
||||
out.append(finding("orphan-page", rel, None, "no other concept and no index.md links to this page (log.md does not count)"))
|
||||
return out
|
||||
|
||||
|
||||
|
||||
def placed_links(doc: Document) -> list[tuple[str, int, str]]:
|
||||
"""(place, file line, target) for every inline markdown link in the body; place is prose, code, heading or footnote.
|
||||
|
||||
Code is what mask_code blanks (inline spans, and blocks fenced with backticks) and every line of a fenced block,
|
||||
backticks or tildes, fences included, also when the fence is indented in a list item or stands behind > in a quote
|
||||
block (K2 point 14). body_links() is the prose and heading/footnote part.
|
||||
"""
|
||||
masked = mask_code(doc.body) # same length as the body, so index by offset
|
||||
out: list[tuple[str, int, str]] = []
|
||||
offset = 0
|
||||
fence = "" # the open fence's marker run, or "" outside a fenced block
|
||||
for i, raw in enumerate(doc.body.split("\n")):
|
||||
f = fence_marker(raw)
|
||||
in_fence = bool(fence or f)
|
||||
if fence and closes_fence(f, raw, fence):
|
||||
fence = ""
|
||||
elif not fence and f:
|
||||
fence = f.group(1)
|
||||
place = "heading" if HEADING.match(raw) else "footnote" if FOOTNOTE_DEF.match(raw.strip()) else "prose"
|
||||
for m in link_matches(raw, masked[offset:offset + len(raw)]):
|
||||
at = offset + m.start()
|
||||
in_code = in_fence or masked[at] != doc.body[at] # the link opens inside code; code inside the anchor is fine
|
||||
out.append(("code" if in_code else place, doc.body_start + i, m.group(2)))
|
||||
offset += len(raw) + 1
|
||||
return out
|
||||
|
||||
|
||||
def concept_inbound(b: Bundle) -> tuple[dict[str, set[str]], list[Finding]]:
|
||||
"""Criterion 6 (b) and (c): inbound concept links per concept page counted from prose only, and misplaced links."""
|
||||
incoming: dict[str, set[str]] = {d.rel: set() for d in b.concepts}
|
||||
misplaced: list[Finding] = []
|
||||
for d in b.concepts:
|
||||
for place, line, target in placed_links(d):
|
||||
if not is_bundle_link(target):
|
||||
continue
|
||||
if place != "prose":
|
||||
misplaced.append(finding("link-misplaced", d.rel, line,
|
||||
f"link to {target!r} sits in {place}; a link belongs in a body sentence (criterion 6 b)"))
|
||||
continue
|
||||
rel = resolve(d.rel, target)
|
||||
if rel in incoming and rel != d.rel:
|
||||
incoming[rel].add(d.rel)
|
||||
return incoming, misplaced
|
||||
|
||||
|
||||
def check_link_structure(b: Bundle) -> list[Finding]:
|
||||
"""link-misplaced (b) of criterion 6; (a) is broken-link in check_links, and the main-page rule is check_main_pages."""
|
||||
return concept_inbound(b)[1]
|
||||
|
||||
|
||||
def link_rule_report(b: Bundle) -> dict[str, Any]:
|
||||
"""Criterion 6 as numbers: N of N concept pages with an inbound link, the breaches of (a) and (b), and counts."""
|
||||
incoming, misplaced = concept_inbound(b)
|
||||
concepts = set(incoming)
|
||||
passed = sum(1 for s in incoming.values() if s)
|
||||
not_to_concept = 0
|
||||
out_degree: dict[str, set[str]] = {rel: set() for rel in concepts}
|
||||
provenance: dict[str, int] = {}
|
||||
for d in b.concepts:
|
||||
for place, _, target in placed_links(d):
|
||||
if place != "prose" or not is_bundle_link(target):
|
||||
continue
|
||||
rel = resolve(d.rel, target)
|
||||
if rel in concepts and rel != d.rel:
|
||||
out_degree[d.rel].add(rel)
|
||||
elif rel not in concepts and b.is_file(rel):
|
||||
not_to_concept += 1
|
||||
srcs = d.frontmatter.get("sources") if d.frontmatter else None
|
||||
for s in srcs if isinstance(srcs, list) else []:
|
||||
value = s.get("x-provenance") if isinstance(s, dict) else None
|
||||
key = value if isinstance(value, str) else "missing"
|
||||
provenance[key] = provenance.get(key, 0) + 1
|
||||
return {
|
||||
"pass": passed,
|
||||
"of": len(concepts),
|
||||
"line": f"link rule: {passed} of {len(concepts)} concept pages have an inbound link",
|
||||
"misplaced": len(misplaced),
|
||||
"broken": sum(f.check == "broken-link" for f in check_links(b)),
|
||||
"not_to_concept": not_to_concept,
|
||||
"max_out_degree": max((len(v) for v in out_degree.values()), default=0),
|
||||
"provenance": dict(sorted(provenance.items())),
|
||||
}
|
||||
|
||||
|
||||
# --- the main-page rule (contract K2, read against K1) -------------------------------------------------------------
|
||||
|
||||
K1_FILE = "source-manifest.json"
|
||||
K1_SCHEMA = "okf-wiki/source-manifest/1"
|
||||
MAIN_FOR = "x-main-for"
|
||||
|
||||
|
||||
class K1Error(Exception):
|
||||
"""The K1 file at the sources root is not valid, so the main-page rule cannot run (K2 point 18)."""
|
||||
|
||||
|
||||
def read_k1(sources: Path) -> dict[str, bool] | None:
|
||||
"""`main_topic` per `path` from <sources>/source-manifest.json (contract K1), read by this module itself and not
|
||||
by the K1 module, which wave 2 changes beside it. None when there is no such file. K1Error when it is not valid:
|
||||
not JSON, `schema` not okf-wiki/source-manifest/1, `sources` not a list, or an entry without `path` (a string) or
|
||||
`main_topic` (a boolean). Unknown keys are ignored (K1 point 10)."""
|
||||
path = sources / K1_FILE
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, ValueError) as exc:
|
||||
raise K1Error(f"{path} is not readable JSON: {exc}") from exc
|
||||
if not isinstance(data, dict) or data.get("schema") != K1_SCHEMA:
|
||||
raise K1Error(f"{path}: schema is not {K1_SCHEMA}")
|
||||
entries = data.get("sources")
|
||||
if not isinstance(entries, list):
|
||||
raise K1Error(f"{path}: sources is not a list")
|
||||
out: dict[str, bool] = {}
|
||||
for i, e in enumerate(entries):
|
||||
if not (isinstance(e, dict) and isinstance(e.get("path"), str) and isinstance(e.get("main_topic"), bool)):
|
||||
raise K1Error(f"{path}: sources[{i}] has no path (a string) or main_topic (a boolean)")
|
||||
out[e["path"]] = e["main_topic"]
|
||||
return out
|
||||
|
||||
|
||||
def main_marks(b: Bundle, k1: dict[str, bool]) -> tuple[dict[str, set[str]], list[Finding]]:
|
||||
"""The valid x-main-for values per page, and one main-page-invalid finding per invalid value (K2 points 1, 2, 10-13).
|
||||
|
||||
A valid value is a string that stands, exactly, among the page's own sources[].resource and is "/" + a path with
|
||||
main_topic true in K1. A value that is not a list, or an empty list, is one finding and gives no role; in a list,
|
||||
each item is judged alone, and a value given twice counts once and gives one finding. A page is a main page when at
|
||||
least one value is valid. A page whose frontmatter does not parse has no role (point 16).
|
||||
"""
|
||||
topics = {"/" + p for p, main in k1.items() if main}
|
||||
valid: dict[str, set[str]] = {}
|
||||
out: list[Finding] = []
|
||||
for d in b.concepts:
|
||||
fm = d.frontmatter
|
||||
if fm is None or MAIN_FOR not in fm:
|
||||
continue
|
||||
value = fm[MAIN_FOR]
|
||||
if not isinstance(value, list):
|
||||
out.append(finding("main-page-invalid", d.rel, None, f"{MAIN_FOR} {value!r} is not a list (K2 point 1)"))
|
||||
continue
|
||||
if not value:
|
||||
out.append(finding("main-page-invalid", d.rel, None,
|
||||
f"{MAIN_FOR} is an empty list; a page that is no main page leaves the key out (K2 point 11)"))
|
||||
continue
|
||||
srcs = fm.get("sources")
|
||||
own = {s["resource"] for s in (srcs if isinstance(srcs, list) else [srcs])
|
||||
if isinstance(s, dict) and isinstance(s.get("resource"), str)}
|
||||
seen: set[str] = set()
|
||||
for v in value:
|
||||
if not isinstance(v, str):
|
||||
why = "is not a string (K2 point 10)"
|
||||
elif v in seen:
|
||||
why = "is listed twice (K2 point 12)"
|
||||
elif v not in own:
|
||||
why = "is not among the page's own sources[].resource (K2 points 1 and 13)"
|
||||
elif v not in topics:
|
||||
why = "is not / + a path with main_topic true in K1 (K2 points 1 and 13)"
|
||||
else:
|
||||
why = ""
|
||||
valid.setdefault(d.rel, set()).add(v)
|
||||
if isinstance(v, str):
|
||||
seen.add(v)
|
||||
if why:
|
||||
out.append(finding("main-page-invalid", d.rel, None, f"{MAIN_FOR} value {v!r} {why}"))
|
||||
return valid, out
|
||||
|
||||
|
||||
def main_holders(valid: dict[str, set[str]], k1: dict[str, bool]) -> dict[str, list[str]]:
|
||||
"""For every K1 path with main_topic true, the pages that list it validly in x-main-for (valid: main_marks)."""
|
||||
return {p: sorted(rel for rel, values in valid.items() if "/" + p in values) for p, main in sorted(k1.items()) if main}
|
||||
|
||||
|
||||
def main_page_links(b: Bundle, mains: set[str]) -> dict[str, set[str]]:
|
||||
"""For every concept page, the main pages whose body links to it (K2 points 3, 14 and 15).
|
||||
|
||||
Only inline links with a relative target count, in prose: paragraphs, list items, table cells and quote blocks
|
||||
under any heading, outside code (inline and fenced), headings and footnote definitions. A link with #anchor counts
|
||||
for its file; a link starting with / does not; a link from the page itself does not. Reference links and autolinks
|
||||
are no inline links, so they do not count.
|
||||
"""
|
||||
linked: dict[str, set[str]] = {d.rel: set() for d in b.concepts}
|
||||
for d in b.concepts:
|
||||
if d.rel not in mains:
|
||||
continue
|
||||
for place, _, target in placed_links(d):
|
||||
if place != "prose" or not is_bundle_link(target) or target.startswith("/"):
|
||||
continue
|
||||
rel = resolve(d.rel, target)
|
||||
if rel in linked and rel != d.rel:
|
||||
linked[rel].add(d.rel)
|
||||
return linked
|
||||
|
||||
|
||||
def check_main_pages(b: Bundle, k1: dict[str, bool]) -> list[Finding]:
|
||||
"""K2 point 5: main-page-missing and main-page-duplicate per main-topic source (the finding's file is its K1 path),
|
||||
main-page-invalid per invalid value, and no-main-page-link per page that is not a main page and has no link from one."""
|
||||
valid, out = main_marks(b, k1)
|
||||
for path, pages in main_holders(valid, k1).items():
|
||||
if not pages:
|
||||
out.append(finding("main-page-missing", path, None,
|
||||
f"main-topic source /{path} has no main page: no concept page lists it in {MAIN_FOR} (K2 point 2)"))
|
||||
elif len(pages) > 1:
|
||||
out.append(finding("main-page-duplicate", path, None,
|
||||
f"main-topic source /{path} has {len(pages)} main pages: {', '.join(pages)} (K2 point 2)"))
|
||||
for rel, mains in sorted(main_page_links(b, set(valid)).items()):
|
||||
if rel not in valid and not mains:
|
||||
out.append(finding("no-main-page-link", rel, None,
|
||||
"no main page links to this page from its body, in a sentence or a list of related concepts; "
|
||||
"index.md, log.md and the page itself do not count (K2 point 3)"))
|
||||
return out
|
||||
|
||||
|
||||
def main_page_report(b: Bundle, k1: dict[str, bool]) -> dict[str, Any]:
|
||||
"""Criterion 6 as numbers (K2 point 7): the main-topic sources with exactly one main page, the pages that are not
|
||||
main pages with a link from one, and the counts of link_rule_report beside them."""
|
||||
valid, _ = main_marks(b, k1)
|
||||
holders = main_holders(valid, k1)
|
||||
sources = sum(1 for pages in holders.values() if len(pages) == 1)
|
||||
linked = main_page_links(b, set(valid))
|
||||
others = [rel for rel in linked if rel not in valid]
|
||||
passed = sum(1 for rel in others if linked[rel])
|
||||
counts = link_rule_report(b)
|
||||
return {
|
||||
"line": f"main pages: {sources} of {len(holders)} sources · linked from a main page: {passed} of {len(others)} pages",
|
||||
"main_pages": {"pass": sources, "of": len(holders)},
|
||||
"linked": {"pass": passed, "of": len(others)},
|
||||
**{k: counts[k] for k in ("misplaced", "broken", "not_to_concept", "max_out_degree", "provenance")},
|
||||
}
|
||||
|
||||
|
||||
def as_events(value: Any) -> list[Any]:
|
||||
"""`verified` as a list of events; a bare mapping is a one-element list (SPEC 5.2)."""
|
||||
if isinstance(value, dict):
|
||||
return [value]
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
return [value]
|
||||
|
||||
|
||||
TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:\d{2})$")
|
||||
|
||||
|
||||
def timestamp_ok(value: Any) -> bool:
|
||||
"""ISO 8601 datetime with an explicit UTC offset (SPEC 5), kept as the text the author wrote (format choice c)."""
|
||||
if not isinstance(value, str) or not TIMESTAMP.match(value):
|
||||
return False
|
||||
try:
|
||||
datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def timestamp_values(fm: dict[str, Any]) -> list[tuple[str, Any]]:
|
||||
"""Every OKF timestamp-valued key present: generated.at, verified[].at, stale_after, sources[].last_modified,
|
||||
and the from/to of usage_window, shared or per source (SPEC 5.1, 5.2, 5.5)."""
|
||||
out: list[tuple[str, Any]] = []
|
||||
|
||||
def window(prefix: str, w: Any) -> None:
|
||||
if isinstance(w, dict):
|
||||
out.extend((f"{prefix}.{k}", w[k]) for k in ("from", "to") if k in w)
|
||||
|
||||
gen = fm.get("generated")
|
||||
if isinstance(gen, dict) and "at" in gen:
|
||||
out.append(("generated.at", gen["at"]))
|
||||
if "verified" in fm:
|
||||
out.extend(("verified[].at", e["at"]) for e in as_events(fm["verified"]) if isinstance(e, dict) and "at" in e)
|
||||
if "stale_after" in fm:
|
||||
out.append(("stale_after", fm["stale_after"]))
|
||||
window("usage_window", fm.get("usage_window"))
|
||||
srcs = fm.get("sources")
|
||||
if isinstance(srcs, list):
|
||||
for s in srcs:
|
||||
if isinstance(s, dict):
|
||||
if "last_modified" in s:
|
||||
out.append(("sources[].last_modified", s["last_modified"]))
|
||||
window("sources[].usage_window", s.get("usage_window"))
|
||||
return out
|
||||
|
||||
|
||||
def check_format(b: Bundle) -> list[Finding]:
|
||||
"""Format choices (a) resource with a leading /, (b) relative body links, (c) timestamps, (e) verified by humans."""
|
||||
out: list[Finding] = []
|
||||
for d in b.concepts:
|
||||
for line, target in body_links(d):
|
||||
if is_bundle_link(target) and target.startswith("/"):
|
||||
out.append(finding("body-link-bundle-absolute", d.rel, line,
|
||||
f"body link {target!r} starts with /; body links are relative (format choice b)"))
|
||||
fm = d.frontmatter
|
||||
if fm is None:
|
||||
continue
|
||||
srcs = fm.get("sources")
|
||||
for s in srcs if isinstance(srcs, list) else []:
|
||||
res = s.get("resource") if isinstance(s, dict) else None
|
||||
if isinstance(res, str) and res and not SCHEME.match(res) and not res.startswith("/"):
|
||||
out.append(finding("resource-not-bundle-absolute", d.rel, None,
|
||||
f"sources[].resource {res!r} has no leading / (format choice a)"))
|
||||
if "verified" in fm:
|
||||
for e in as_events(fm["verified"]):
|
||||
by = e.get("by") if isinstance(e, dict) else None
|
||||
if not (isinstance(by, str) and by.startswith("human:") and len(by) > len("human:")):
|
||||
out.append(finding("verified-not-human", d.rel, None,
|
||||
f"verified by {by!r}: only a human:<id> may write verified (format choice e)"))
|
||||
for key, value in timestamp_values(fm):
|
||||
if not timestamp_ok(value):
|
||||
out.append(finding("timestamp-form", d.rel, None,
|
||||
f"{key} {value!r} is not an ISO 8601 datetime with an explicit UTC offset (SPEC 5)"))
|
||||
return out
|
||||
|
||||
|
||||
def check_citations(b: Bundle) -> list[Finding]:
|
||||
"""Criterion 1 and format choice f: sentence -> footnote -> sources[].id -> verbatim quote at its locator."""
|
||||
out: list[Finding] = []
|
||||
source_lines: dict[str, list[str] | None] = {}
|
||||
for d in b.concepts:
|
||||
for block in prose_blocks(d):
|
||||
if LINK_ONLY.match(mask_code(block.text)):
|
||||
continue # a pure link list is not a claim
|
||||
for s in sentences(block.text):
|
||||
if not FOOTNOTE_REF.search(s):
|
||||
out.append(finding("uncited-sentence", d.rel, block.line, f"sentence without a footnote: {s[:80]!r}"))
|
||||
fm = d.frontmatter
|
||||
if fm is None:
|
||||
continue # without parseable frontmatter there are no sources to join to (spec11-1 reports it)
|
||||
srcs = fm.get("sources", [])
|
||||
entries = srcs if isinstance(srcs, list) else [srcs]
|
||||
ids: list[str] = []
|
||||
for s in entries:
|
||||
sid = s.get("id") if isinstance(s, dict) else None
|
||||
if not isinstance(sid, str) or not sid.strip():
|
||||
out.append(finding("source-missing-id", d.rel, None, "a sources entry has no id (format choice f)"))
|
||||
else:
|
||||
ids.append(sid)
|
||||
for sid in sorted({i for i in ids if ids.count(i) > 1}):
|
||||
out.append(finding("source-duplicate-id", d.rel, None, f"sources[].id {sid!r} is used more than once (format choice f)"))
|
||||
known = set(ids)
|
||||
labels: dict[str, int] = {}
|
||||
for i, raw in masked_lines(d.body):
|
||||
m = FOOTNOTE_DEF.match(raw.strip())
|
||||
found = [m.group(1)] if m else FOOTNOTE_REF.findall(raw)
|
||||
for label in found:
|
||||
labels.setdefault(label, d.body_start + i)
|
||||
for label, line in sorted(labels.items(), key=lambda kv: kv[1]):
|
||||
if label not in known:
|
||||
out.append(finding("footnote-unknown-id", d.rel, line, f"footnote [^{label}] names no sources[].id"))
|
||||
for s in entries:
|
||||
if isinstance(s, dict):
|
||||
out.extend(check_quote(b, d, s, source_lines))
|
||||
return out
|
||||
|
||||
|
||||
PLACE_KEYS = frozenset({"page", "paragraph", "sheet", "cell", "slide", "notes"})
|
||||
|
||||
|
||||
def check_quote(b: Bundle, d: Document, s: dict[str, Any], cache: dict[str, list[str] | None]) -> list[Finding]:
|
||||
sid = s.get("id", "?")
|
||||
quote, locator, res = s.get("x-quote"), s.get("x-locator"), s.get("resource")
|
||||
if not isinstance(quote, str) or not quote:
|
||||
return [finding("quote-missing", d.rel, None, f"source {sid!r} has no x-quote")]
|
||||
if not (isinstance(locator, dict) and "line" in locator and set(locator) - {"line"} <= PLACE_KEYS
|
||||
and type(locator["line"]) is int and locator["line"] > 0):
|
||||
return [finding("locator-unsupported", d.rel, None, f"source {sid!r}: x-locator {locator!r} is not {{ line: N }}"
|
||||
" with at most a page, paragraph, sheet and cell, or slide and notes")]
|
||||
if not isinstance(res, str) or not res or SCHEME.match(res):
|
||||
return [finding("source-unreadable", d.rel, None, f"source {sid!r}: resource {res!r} is not a raw-source path")]
|
||||
rel = resolve(d.rel, res)
|
||||
if not rel:
|
||||
return [finding("source-unreadable", d.rel, None, f"source {sid!r}: resource {res!r} leaves the raw-source root")]
|
||||
if rel not in cache:
|
||||
try:
|
||||
cache[rel] = (b.sources / rel).read_text(encoding="utf-8").splitlines()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
cache[rel] = None
|
||||
lines = cache[rel]
|
||||
if lines is None:
|
||||
return [finding("source-unreadable", d.rel, None, f"source {sid!r}: cannot read {res!r} as UTF-8 text")]
|
||||
n = locator["line"]
|
||||
if n <= len(lines) and quote in lines[n - 1]:
|
||||
place = {k: v for k, v in locator.items() if k != "line"}
|
||||
places = b.places(rel)
|
||||
frozen = places[n - 1] if places is not None and n <= len(places) else None
|
||||
if not place_fits(place, frozen):
|
||||
return [finding("quote-wrong-place", d.rel, None, f"source {sid!r}: line {n} of {res!r} is at "
|
||||
f"{frozen or 'no place'} in the original, not {place}")]
|
||||
return []
|
||||
if any(quote in line for line in lines):
|
||||
return [finding("quote-wrong-locator", d.rel, None, f"source {sid!r}: the quote is in {res!r}, but not on line {n}")]
|
||||
return [finding("quote-not-verbatim", d.rel, None, f"source {sid!r}: the quote is not verbatim anywhere in {res!r}")]
|
||||
|
||||
|
||||
def audit(bundle: Path, sources: Path) -> list[Finding]:
|
||||
"""Every finding on the bundle; the main-page rule only when SOURCES holds a K1 file (K1Error when it is invalid)."""
|
||||
k1 = read_k1(sources)
|
||||
return check_bundle(load_bundle(bundle, sources), k1)
|
||||
|
||||
|
||||
def check_bundle(b: Bundle, k1: dict[str, bool] | None) -> list[Finding]:
|
||||
found = (check_spec11_concepts(b) + check_spec11_index(b) + check_spec11_log(b)
|
||||
+ check_links(b) + check_link_structure(b) + check_citations(b) + check_format(b))
|
||||
if k1 is not None:
|
||||
found += check_main_pages(b, k1)
|
||||
return sorted(found, key=lambda f: (GROUPS.index(f.group), f.file, f.line or 0, f.check, f.message))
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = sys.argv[1:] if argv is None else argv
|
||||
if len(args) != 2:
|
||||
print("usage: python3 -m okfwiki.audit BUNDLE SOURCES", file=sys.stderr)
|
||||
return 2
|
||||
bundle, sources = Path(args[0]), Path(args[1])
|
||||
for p, what in ((bundle, "bundle"), (sources, "raw-source root")):
|
||||
if not p.is_dir():
|
||||
print(f"audit cannot run: {what} {p} is not a directory", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
k1 = read_k1(sources)
|
||||
except K1Error as exc:
|
||||
print(f"audit cannot run: {exc}; the main-page rule needs a valid K1 file (K2 points 6 and 18)", file=sys.stderr)
|
||||
return 2
|
||||
if k1 is None:
|
||||
print(f"audit cannot run: no {K1_FILE} at the raw-source root {sources}; the main-page rule needs K1 (K2 point 6)",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
b = load_bundle(bundle, sources)
|
||||
if not any(k1.values()) and not b.concepts:
|
||||
print(f"audit cannot run: {K1_FILE} lists no main-topic source and the bundle has no concept page, so the "
|
||||
"main-page rule has no denominator", file=sys.stderr)
|
||||
return 2
|
||||
found = check_bundle(b, k1)
|
||||
summary = {g: sum(f.group == g for f in found) for g in GROUPS}
|
||||
print(json.dumps({"findings": [asdict(f) for f in found], "summary": summary, "link_rule": main_page_report(b, k1)},
|
||||
ensure_ascii=False, indent=1))
|
||||
return 1 if found else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
467
okfwiki/claims.py
Normal file
467
okfwiki/claims.py
Normal file
|
|
@ -0,0 +1,467 @@
|
|||
"""The claim list behind PLAN.md criterion 1 ("sitat ved plassering") and 2 ("slutning støttet"), and the reading
|
||||
layout for pages (K6 in the wave-2 contracts). It calls no model.
|
||||
|
||||
The denominator comes from the pages, not from the footnotes: every sentence in the body of a concept page is a claim.
|
||||
Headings and pure link lists are not claims; they are listed as exceptions so the count can be checked by hand. Every
|
||||
claim has a stable id "<page>#<key>", the key being klink's sentence key (okfwiki.klink.page_sentences) for the same
|
||||
sentence at the same place, so the id survives a new assembly of the page; a claim klink does not see is listed as
|
||||
unkeyed and counts as failing, never as a claim without an id. Every claim carries excerpt_sha256 (the excerpts it shows
|
||||
the reader, in order) and its provenance mark: the strongest x-provenance among the entries it cites, missing > merged >
|
||||
inferred > documented, "missing" when an entry has no valid mark or the claim cites none (so it is read, never sampled).
|
||||
|
||||
list BUNDLE SOURCES --out CLAIMS [--pairs PAIRS]
|
||||
Criterion 1: a claim counts when it has at least one footnote and every footnote names a sources[].id whose
|
||||
x-quote stands verbatim at its x-locator (the auditor's own quote check). With --pairs, the reader set
|
||||
({"pairs": [{id, claim, excerpts: [{file, line, text}]}]}); a claim with no quote to show is left out.
|
||||
calls --claims CLAIMS --pairs PAIRS --out CALLS [--ids IDS] [--size N]
|
||||
The call layout (okf-wiki/reader-calls/1) from the pages: the claims of one page stay together in calls of
|
||||
at most ten (at most N with --size, 1 to 10), in page order; only claims in PAIRS, and with --ids only
|
||||
those listed.
|
||||
sample --claims CLAIMS --judged JUDGED --part-root ROOT --seed N --min 300 --out IDS
|
||||
The claims to read after the first part: a claim is judged when its id and excerpt_sha256 stand in JUDGED
|
||||
(the claim list of an earlier reading). Read are every claim not judged that cites a source of the part
|
||||
(ROOT's source-manifest.json) or is marked inferred, merged or missing, and a seeded draw of at least --min
|
||||
of the others. Written with the seed, the sha256 of the inputs and of the ids, and the error bound.
|
||||
sample --claims CLAIMS --of-all --seed N --min 300 --out IDS
|
||||
The reading after a build changes (the operator 13.11.2026): a seeded draw of --min among every claim,
|
||||
whatever its mark and whether it was judged before, and nothing added to it.
|
||||
score --claims CLAIMS --verdicts VERDICTS [--ids IDS]
|
||||
Criterion 2 over the same claims (or the ids): a claim counts when its verdict is "supported"; a missing or
|
||||
invalid verdict counts as not supported (fail-closed). Not supported is counted per provenance mark, and for
|
||||
a sample the upper one-sided 95 % Clopper-Pearson bound is printed (0 errors in 299 is under 1 %).
|
||||
|
||||
Exit 0 = N of N (or written), 1 = at least one claim does not count (or nothing to write), 2 = could not run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import random
|
||||
import sys
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from okfwiki.audit import (
|
||||
FOOTNOTE_REF,
|
||||
LINK_ONLY,
|
||||
Bundle,
|
||||
Document,
|
||||
body_headings,
|
||||
check_quote,
|
||||
load_bundle,
|
||||
mask_code,
|
||||
prose_blocks,
|
||||
sentence_spans,
|
||||
)
|
||||
from okfwiki.klink import key_text, page_sentences
|
||||
from okfwiki.reader import SUPPORTED, load_pairs
|
||||
|
||||
PROVENANCE = ("documented", "inferred", "merged")
|
||||
MISSING = "missing"
|
||||
STRENGTH = {"documented": 0, "inferred": 1, "merged": 2, MISSING: 3}
|
||||
MUST_READ = ("inferred", "merged", MISSING)
|
||||
CALL_SIZE = 10
|
||||
SAMPLE_MIN = 300
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Excerpt:
|
||||
file: str
|
||||
line: int
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Claim:
|
||||
id: str
|
||||
page: str
|
||||
line: int
|
||||
claim: str
|
||||
footnotes: list[str]
|
||||
excerpts: list[Excerpt]
|
||||
quote_ok: bool
|
||||
excerpt_sha256: str = ""
|
||||
provenance: str = MISSING
|
||||
resources: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Exempt:
|
||||
page: str
|
||||
line: int
|
||||
kind: str # "heading" or "link-list"
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Unkeyed:
|
||||
page: str
|
||||
line: int
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClaimList:
|
||||
claims: list[Claim] = field(default_factory=list)
|
||||
exceptions: list[Exempt] = field(default_factory=list)
|
||||
unkeyed: list[Unkeyed] = field(default_factory=list)
|
||||
|
||||
|
||||
def excerpt_digest(excerpts: list[Excerpt]) -> str:
|
||||
"""sha256 of the excerpts a claim shows the reader, in order (file, line, text)."""
|
||||
rows = [[e.file, e.line, e.text] for e in excerpts]
|
||||
return hashlib.sha256(json.dumps(rows, ensure_ascii=False, separators=(",", ":")).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def provenance_of(doc: Document, labels: list[str]) -> str:
|
||||
"""The strongest mark among the cited entries; missing when an entry has no valid mark or nothing is cited."""
|
||||
by_id = _sources_by_id(doc)
|
||||
marks = [by_id[x].get("x-provenance") if x in by_id else None for x in labels] or [None]
|
||||
return max((m if isinstance(m, str) and m in PROVENANCE else MISSING for m in marks), key=STRENGTH.__getitem__)
|
||||
|
||||
|
||||
def _digest(text: str) -> str:
|
||||
return hashlib.sha256(key_text(text).encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def _sources_by_id(doc: Document) -> dict[str, dict[str, Any]]:
|
||||
fm = doc.frontmatter or {}
|
||||
srcs = fm.get("sources", [])
|
||||
entries = srcs if isinstance(srcs, list) else [srcs]
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for s in entries:
|
||||
if isinstance(s, dict) and isinstance(s.get("id"), str):
|
||||
out.setdefault(s["id"], s)
|
||||
return out
|
||||
|
||||
|
||||
def _judge(b: Bundle, doc: Document, labels: list[str], cache: dict[str, list[str] | None]) -> tuple[list[Excerpt], bool]:
|
||||
"""The excerpts a claim cites, and whether every one of them stands verbatim at its locator."""
|
||||
by_id = _sources_by_id(doc)
|
||||
excerpts: list[Excerpt] = []
|
||||
ok = bool(labels)
|
||||
for label in labels:
|
||||
s = by_id.get(label)
|
||||
if s is None:
|
||||
ok = False
|
||||
continue
|
||||
if check_quote(b, doc, s, cache):
|
||||
ok = False
|
||||
quote, locator, res = s.get("x-quote"), s.get("x-locator"), s.get("resource")
|
||||
if isinstance(quote, str) and quote and isinstance(res, str) and isinstance(locator, dict) \
|
||||
and type(locator.get("line")) is int:
|
||||
e = Excerpt(PurePosixPath(res).name, locator["line"], quote)
|
||||
if e not in excerpts:
|
||||
excerpts.append(e)
|
||||
return excerpts, ok
|
||||
|
||||
|
||||
def extract(bundle: Path, sources: Path) -> ClaimList:
|
||||
b = load_bundle(bundle, sources)
|
||||
out = ClaimList()
|
||||
cache: dict[str, list[str] | None] = {}
|
||||
for doc in b.concepts:
|
||||
keys = [x.key for x in page_sentences((bundle / doc.rel).read_text(encoding="utf-8"))]
|
||||
at = 0
|
||||
for line, _level, text in body_headings(doc):
|
||||
out.exceptions.append(Exempt(doc.rel, line, "heading", text))
|
||||
for block in prose_blocks(doc):
|
||||
if LINK_ONLY.match(mask_code(block.text)):
|
||||
out.exceptions.append(Exempt(doc.rel, block.line, "link-list", block.text))
|
||||
continue
|
||||
for a, z in sentence_spans(block.text):
|
||||
sentence = block.text[a:z]
|
||||
labels: list[str] = []
|
||||
for label in FOOTNOTE_REF.findall(mask_code(sentence)):
|
||||
if label not in labels:
|
||||
labels.append(label)
|
||||
text = FOOTNOTE_REF.sub("", sentence).strip()
|
||||
digest = _digest(sentence)
|
||||
found = next((i for i in range(at, len(keys)) if keys[i].rsplit("-", 1)[0] == digest), None)
|
||||
if found is None:
|
||||
out.unkeyed.append(Unkeyed(doc.rel, block.line, text))
|
||||
continue
|
||||
at = found + 1
|
||||
excerpts, ok = _judge(b, doc, labels, cache)
|
||||
resources = [str(r) for r in (_sources_by_id(doc).get(x, {}).get("resource") for x in labels)
|
||||
if isinstance(r, str)]
|
||||
out.claims.append(Claim(f"{doc.rel}#{keys[found]}", doc.rel, block.line, text, labels, excerpts, ok,
|
||||
excerpt_digest(excerpts), provenance_of(doc, labels),
|
||||
list(dict.fromkeys(resources))))
|
||||
return out
|
||||
|
||||
|
||||
Tally = tuple[int, int]
|
||||
|
||||
|
||||
def _tally(items: list[tuple[list[str], bool]]) -> dict[str, Tally]:
|
||||
counts: dict[str, list[int]] = {}
|
||||
for keys, ok in items:
|
||||
for k in keys:
|
||||
c = counts.setdefault(k, [0, 0])
|
||||
c[0] += int(ok)
|
||||
c[1] += 1
|
||||
return {k: (v[0], v[1]) for k, v in sorted(counts.items())}
|
||||
|
||||
|
||||
def _report(found: ClaimList, ok: dict[str, bool], cs: list[Claim] | None = None, extra: int = 0) -> dict[str, Any]:
|
||||
"""Totals per page, per cited file and per provenance mark; `extra` claims that cannot count (unkeyed claims,
|
||||
ids that are not claims) are added to the denominator as failing."""
|
||||
cs = found.claims if cs is None else cs
|
||||
return {
|
||||
"total": (sum(ok[c.id] for c in cs), len(cs) + extra),
|
||||
"per_page": _tally([([c.page], ok[c.id]) for c in cs]),
|
||||
"per_file": _tally([(sorted({e.file for e in c.excerpts}), ok[c.id]) for c in cs]),
|
||||
"not_supported_per_mark": {k: (b - a, b)
|
||||
for k, (a, b) in _tally([([c.provenance], ok[c.id]) for c in cs]).items()},
|
||||
"not_supported": sorted(c.id for c in cs if not ok[c.id]),
|
||||
}
|
||||
|
||||
|
||||
def criterion_1(found: ClaimList) -> dict[str, Any]:
|
||||
return _report(found, {c.id: c.quote_ok for c in found.claims}, extra=len(found.unkeyed))
|
||||
|
||||
|
||||
def criterion_2(found: ClaimList, verdicts: dict[str, Any], ids: list[str] | None = None) -> dict[str, Any]:
|
||||
given: dict[str, list[str]] = {}
|
||||
for v in verdicts.get("verdicts", []) if isinstance(verdicts, dict) else []:
|
||||
if isinstance(v, dict) and isinstance(v.get("id"), str):
|
||||
given.setdefault(v["id"], []).append(str(v.get("verdict")))
|
||||
ok = {c.id: given.get(c.id) == [SUPPORTED] for c in found.claims}
|
||||
if ids is None:
|
||||
return _report(found, ok, extra=len(found.unkeyed))
|
||||
wanted = set(ids)
|
||||
chosen = [c for c in found.claims if c.id in wanted]
|
||||
return _report(found, ok, chosen, extra=len(wanted) - len(chosen))
|
||||
|
||||
|
||||
def error_bound(k: int, n: int) -> float:
|
||||
"""The upper one-sided 95 % Clopper-Pearson bound on the error rate after k errors in n: the p at which k or fewer
|
||||
errors in n have probability 0.05. For k = 0 it is 1 - 0.05 ** (1 / n)."""
|
||||
if n <= 0 or k >= n:
|
||||
return 1.0
|
||||
|
||||
def at_most_k(p: float) -> float:
|
||||
return sum(math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1)
|
||||
+ i * math.log(p) + (n - i) * math.log1p(-p)) for i in range(k + 1))
|
||||
|
||||
lo, hi = 0.0, 1.0
|
||||
for _ in range(200):
|
||||
mid = (lo + hi) / 2
|
||||
if mid <= 0.0 or at_most_k(mid) > 0.05:
|
||||
lo = mid
|
||||
else:
|
||||
hi = mid
|
||||
return hi
|
||||
|
||||
|
||||
def write_pairs(found: ClaimList, out: Path) -> None:
|
||||
pairs = [{"id": c.id, "claim": c.claim, "excerpts": [asdict(e) for e in c.excerpts]}
|
||||
for c in found.claims if c.excerpts]
|
||||
out.write_text(json.dumps({"schema": "okf-wiki/claim-pairs/1", "pairs": pairs}, indent=1, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8")
|
||||
|
||||
|
||||
def load_claims(path: Path) -> ClaimList:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
found = ClaimList()
|
||||
for c in data["claims"]:
|
||||
excerpts = [Excerpt(**e) for e in c["excerpts"]]
|
||||
found.claims.append(Claim(c["id"], c["page"], c["line"], c["claim"], list(c["footnotes"]), excerpts,
|
||||
bool(c["quote_ok"]), str(c.get("excerpt_sha256") or excerpt_digest(excerpts)),
|
||||
str(c.get("provenance", MISSING)), list(c.get("resources", []))))
|
||||
for e in data.get("exceptions", []):
|
||||
found.exceptions.append(Exempt(**e))
|
||||
for u in data.get("unkeyed", []):
|
||||
found.unkeyed.append(Unkeyed(**u))
|
||||
return found
|
||||
|
||||
|
||||
_load = load_claims
|
||||
|
||||
|
||||
def _sha(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _dump(doc: Any) -> str:
|
||||
return json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def _ids(path: Path) -> list[str]:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
ids = data.get("ids") if isinstance(data, dict) else None
|
||||
if not isinstance(ids, list) or not all(isinstance(i, str) for i in ids):
|
||||
raise ValueError(f"{path} has no list of ids")
|
||||
return ids
|
||||
|
||||
|
||||
def _print(name: str, report: dict[str, Any]) -> None:
|
||||
x, n = report["total"]
|
||||
print(f"{name}: {x} of {n}")
|
||||
for key in ("per_page", "per_file"):
|
||||
print(f" {key.replace('_', ' ')}:")
|
||||
for k, (a, b) in report[key].items():
|
||||
print(f" {k}: {a} of {b}")
|
||||
|
||||
|
||||
def cmd_list(args: argparse.Namespace) -> int:
|
||||
bundle, sources = Path(args.bundle), Path(args.sources)
|
||||
for p, what in ((bundle, "bundle"), (sources, "raw-source root")):
|
||||
if not p.is_dir():
|
||||
print(f"claims cannot run: {what} {p} is not a directory", file=sys.stderr)
|
||||
return 2
|
||||
found = extract(bundle, sources)
|
||||
doc = {"schema": "okf-wiki/claims/1",
|
||||
"claims": [asdict(c) for c in found.claims],
|
||||
"exceptions": [asdict(e) for e in found.exceptions],
|
||||
"unkeyed": [asdict(u) for u in found.unkeyed]}
|
||||
Path(args.out).write_text(json.dumps(doc, indent=1, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
if args.pairs:
|
||||
write_pairs(found, Path(args.pairs))
|
||||
report = criterion_1(found)
|
||||
_print("criterion 1, quote at locator", report)
|
||||
kinds = {k: sum(e.kind == k for e in found.exceptions) for k in ("heading", "link-list")}
|
||||
print(f"exceptions (not claims): {kinds['heading']} headings, {kinds['link-list']} link lists")
|
||||
print(f"claims without a sentence key (counted as failing): {len(found.unkeyed)}")
|
||||
x, n = report["total"]
|
||||
return 0 if n and x == n else 1
|
||||
|
||||
|
||||
def cmd_calls(args: argparse.Namespace) -> int:
|
||||
size: int = args.size
|
||||
if not 1 <= size <= CALL_SIZE:
|
||||
raise ValueError(f"--size {size} is not a whole number from 1 to {CALL_SIZE}; nothing written")
|
||||
found = load_claims(Path(args.claims))
|
||||
pairs_path = Path(args.pairs)
|
||||
with_pair = {p.id for p in load_pairs(pairs_path)}
|
||||
wanted = set(_ids(Path(args.ids))) if args.ids else None
|
||||
chosen = [c for c in found.claims if (wanted is None or c.id in wanted) and c.id in with_pair]
|
||||
left_out = (len(wanted) if wanted is not None else len(found.claims)) - len(chosen)
|
||||
pages: dict[str, list[str]] = {}
|
||||
for c in chosen:
|
||||
pages.setdefault(c.page, []).append(c.id)
|
||||
groups = [ids[i : i + size] for ids in pages.values() for i in range(0, len(ids), size)]
|
||||
if not groups:
|
||||
print("calls: no claim to lay out", file=sys.stderr)
|
||||
return 1
|
||||
layout = {"call_size_max": size, "calls": [{"call": f"k{n:03d}", "ids": g} for n, g in enumerate(groups, 1)],
|
||||
"schema": "okf-wiki/reader-calls/1", "set_sha256": _sha(pairs_path)}
|
||||
Path(args.out).write_text(_dump(layout), encoding="utf-8")
|
||||
print(f"calls: {len(chosen)} claims from {len(pages)} pages in {len(groups)} calls; "
|
||||
f"{left_out} left out (no excerpt to show, or not a claim)")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_sample(args: argparse.Namespace) -> int:
|
||||
if args.min < SAMPLE_MIN:
|
||||
raise ValueError(f"--min {args.min} is under {SAMPLE_MIN}, the least sample K6 point 4 allows")
|
||||
claims_path = Path(args.claims)
|
||||
found = load_claims(claims_path)
|
||||
judged_sha: str | None = None
|
||||
manifest_sha: str | None = None
|
||||
if args.of_all:
|
||||
if args.judged or args.part_root:
|
||||
raise ValueError("--of-all draws among every claim and takes no --judged or --part-root")
|
||||
open_, must = list(found.claims), []
|
||||
else:
|
||||
if not args.judged or not args.part_root:
|
||||
raise ValueError("sample needs --judged and --part-root, or --of-all to draw among every claim")
|
||||
judged_path, manifest = Path(args.judged), Path(args.part_root) / "source-manifest.json"
|
||||
part = {str(s["path"]) for s in json.loads(manifest.read_text(encoding="utf-8"))["sources"]}
|
||||
done = {(c.id, c.excerpt_sha256) for c in load_claims(judged_path).claims}
|
||||
open_ = [c for c in found.claims if (c.id, c.excerpt_sha256) not in done]
|
||||
must = [c.id for c in open_ if c.provenance in MUST_READ or any(r.lstrip("/") in part for r in c.resources)]
|
||||
judged_sha, manifest_sha = _sha(judged_path), _sha(manifest)
|
||||
others = [c.id for c in open_ if c.id not in set(must)]
|
||||
drawn = set(random.Random(args.seed).sample(others, min(len(others), args.min)))
|
||||
sample = [i for i in others if i in drawn]
|
||||
keep = set(must) | drawn
|
||||
ids = [c.id for c in found.claims if c.id in keep]
|
||||
doc = {"schema": "okf-wiki/claim-sample/1", "seed": args.seed, "min": args.min, "of_all": bool(args.of_all),
|
||||
"claims_sha256": _sha(claims_path), "judged_sha256": judged_sha,
|
||||
"part_manifest_sha256": manifest_sha,
|
||||
"counts": {"claims": len(found.claims), "judged": len(found.claims) - len(open_), "must_read": len(must),
|
||||
"others": len(others), "sample": len(sample)},
|
||||
"must_read": must, "sample": sample, "ids": ids,
|
||||
"ids_sha256": hashlib.sha256("\n".join(ids).encode("utf-8")).hexdigest(),
|
||||
"error_bound_at_zero": error_bound(0, len(sample))}
|
||||
Path(args.out).write_text(_dump(doc), encoding="utf-8")
|
||||
print(f"sample (seed {args.seed}): {len(found.claims)} claims, {len(found.claims) - len(open_)} judged, "
|
||||
f"{len(must)} must be read, {len(sample)} drawn of {len(others)}; read {len(ids)}")
|
||||
print(f"0 not supported in the sample would bound the error at {error_bound(0, len(sample)):.2%} "
|
||||
"(one-sided Clopper-Pearson, 95 %)")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_score(args: argparse.Namespace) -> int:
|
||||
try:
|
||||
found = load_claims(Path(args.claims))
|
||||
verdicts = json.loads(Path(args.verdicts).read_text(encoding="utf-8"))
|
||||
ids = _ids(Path(args.ids)) if args.ids else None
|
||||
sample = json.loads(Path(args.ids).read_text(encoding="utf-8")).get("sample") if args.ids else None
|
||||
except (OSError, ValueError, KeyError, TypeError) as exc:
|
||||
print(f"score cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
report = criterion_2(found, verdicts, ids)
|
||||
_print("criterion 2, supported", report)
|
||||
marks = ", ".join(f"{k} {a} of {b}" for k, (a, b) in report["not_supported_per_mark"].items())
|
||||
print(f"not supported per provenance mark: {marks or 'none'}")
|
||||
if isinstance(sample, list):
|
||||
k = len(set(report["not_supported"]) & set(sample)) + len(set(sample) - {c.id for c in found.claims})
|
||||
print(f"{k} not supported in a sample of {len(sample)}: upper bound {error_bound(k, len(sample)):.2%} "
|
||||
"(one-sided Clopper-Pearson, 95 %)")
|
||||
if report["not_supported"]:
|
||||
print("not supported: " + " ".join(report["not_supported"]))
|
||||
x, n = report["total"]
|
||||
return 0 if n and x == n else 1
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
ls = sub.add_parser("list", help="list the claims and count criterion 1")
|
||||
ls.add_argument("bundle")
|
||||
ls.add_argument("sources")
|
||||
ls.add_argument("--out", required=True)
|
||||
ls.add_argument("--pairs")
|
||||
ca = sub.add_parser("calls", help="lay the claims out in calls of at most ten per page")
|
||||
ca.add_argument("--claims", required=True)
|
||||
ca.add_argument("--pairs", required=True)
|
||||
ca.add_argument("--out", required=True)
|
||||
ca.add_argument("--ids")
|
||||
ca.add_argument("--size", type=int, default=CALL_SIZE,
|
||||
help=f"at most this many claims per call, 1 to {CALL_SIZE} (default {CALL_SIZE})")
|
||||
sa = sub.add_parser("sample", help="the claims to read: the part, inferred, merged and a seeded draw")
|
||||
sa.add_argument("--claims", required=True)
|
||||
sa.add_argument("--judged")
|
||||
sa.add_argument("--part-root")
|
||||
sa.add_argument("--of-all", action="store_true", help="draw among every claim and add nothing")
|
||||
sa.add_argument("--seed", type=int, required=True)
|
||||
sa.add_argument("--min", type=int, required=True)
|
||||
sa.add_argument("--out", required=True)
|
||||
sc = sub.add_parser("score", help="count criterion 2 from a reader's verdicts")
|
||||
sc.add_argument("--claims", required=True)
|
||||
sc.add_argument("--verdicts", required=True)
|
||||
sc.add_argument("--ids")
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
commands = {"list": cmd_list, "calls": cmd_calls, "sample": cmd_sample, "score": cmd_score}
|
||||
if args.command in ("list", "calls", "sample"):
|
||||
try:
|
||||
return commands[args.command](args)
|
||||
except (OSError, ValueError, KeyError, TypeError) as exc:
|
||||
print(f"claims cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
return commands[args.command](args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
121
okfwiki/common.py
Normal file
121
okfwiki/common.py
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
"""Helpers shared by the okf-wiki tools. No model call (CLAUDE.md rule 1)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# A release note's version heading: "1.2.3" or "v1.2.3", at least three parts.
|
||||
VERSION_HEADING = re.compile(r"^v?(\d+(?:\.\d+){2,})$")
|
||||
# A line that is only one HTML or MDX tag, such as <Note> or </Steps>: layout, not content. The tag name is letters,
|
||||
# digits, ".", "_" or "-", followed by whitespace, "/" or ">", so a link in angle brackets (<https://...>) is not a tag.
|
||||
HTML_TAG_ONLY = re.compile(r"^\s*</?[A-Za-z][A-Za-z0-9._-]*(?:\s[^<>]*)?/?>\s*$")
|
||||
|
||||
|
||||
class RunError(Exception):
|
||||
"""The command cannot run as asked; nothing was written. A command reports it with exit 2."""
|
||||
|
||||
|
||||
def write_new(path: Path, text: str) -> None:
|
||||
"""Write text to a file that does not exist yet; an existing file is never overwritten."""
|
||||
if path.exists():
|
||||
raise RunError(f"{path} exists; refusing to overwrite")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def jaccard(a: set[str], b: set[str]) -> float:
|
||||
return len(a & b) / len(a | b) if a | b else 0.0
|
||||
|
||||
|
||||
# Letters that NFKD does not split into an ASCII letter and a mark, spelled out as Norwegian and German write them.
|
||||
SPELLED = {"æ": "ae", "ø": "oe", "å": "aa", "ß": "ss", "œ": "oe", "þ": "th", "ð": "d", "ł": "l", "đ": "d"}
|
||||
|
||||
|
||||
def ascii_fold(text: str) -> str:
|
||||
"""Lower-case text with every letter spelled in ASCII where it can be: æ ø å as ae oe aa, an accent dropped
|
||||
(é -> e); what has no ASCII spelling is left as it is."""
|
||||
out = []
|
||||
for ch in unicodedata.normalize("NFC", text).lower():
|
||||
if ch in SPELLED:
|
||||
out.append(SPELLED[ch])
|
||||
else:
|
||||
out.append("".join(c for c in unicodedata.normalize("NFKD", ch) if not unicodedata.combining(c)))
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def slug(text: str) -> str:
|
||||
"""Lower-case letters and digits joined by single hyphens, at most 60 characters; "x" when nothing is left.
|
||||
Letters outside ASCII are spelled out first (ascii_fold), so "Årlig" is "aarlig", not "rlig"."""
|
||||
return re.sub(r"[^a-z0-9]+", "-", ascii_fold(text)).strip("-")[:60] or "x"
|
||||
|
||||
|
||||
def place_words(locator: dict[str, Any]) -> str:
|
||||
"""Where a quote stands, as a reader names it: page 3, paragraph 4, sheet Plan, cell B7, slide 2 (notes), or line N
|
||||
when the source is a text file."""
|
||||
if "page" in locator:
|
||||
return f"page {locator['page']}"
|
||||
if "paragraph" in locator:
|
||||
return f"paragraph {locator['paragraph']}"
|
||||
if "cell" in locator:
|
||||
cell = str(locator["cell"])
|
||||
return f"sheet {locator.get('sheet', '')}, {'cells' if ':' in cell else 'cell'} {cell}"
|
||||
if "slide" in locator:
|
||||
return f"slide {locator['slide']}" + (" (notes)" if locator.get("notes") else "")
|
||||
return f"line {locator.get('line')}"
|
||||
|
||||
|
||||
def place_at(place: dict[str, Any] | None, line: str, quote: str) -> dict[str, Any]:
|
||||
"""The place in the original of a quote on a line of a frozen copy, from that line's entry in K1 `places`. A
|
||||
spreadsheet row's entry lists its cells with the offset each starts at, so the quote gets the cell, or the range
|
||||
of cells, it covers; a quote not on the line gets the whole row. Any other entry is the place as it stands."""
|
||||
if not place:
|
||||
return {}
|
||||
if "cells" not in place:
|
||||
return dict(place)
|
||||
cells = [(str(ref), int(at)) for ref, at in place["cells"]]
|
||||
start = line.find(quote) if quote else -1
|
||||
end = start + len(quote)
|
||||
if start < 0:
|
||||
covered = [ref for ref, _ in cells]
|
||||
else:
|
||||
bounds = [at for _, at in cells[1:]] + [len(line) + 1]
|
||||
covered = [ref for (ref, at), stop in zip(cells, bounds) if at < end and stop > start]
|
||||
cell = covered[0] if len(covered) == 1 else f"{covered[0]}:{covered[-1]}"
|
||||
return {"sheet": place["sheet"], "cell": cell}
|
||||
|
||||
|
||||
def place_fits(cited: dict[str, Any], frozen: dict[str, Any] | None) -> bool:
|
||||
"""Whether a cited place (an x-locator without its line) is the frozen line's place: for a spreadsheet row, the
|
||||
same sheet and a cell, or a range from one cell to a later one, of that row; otherwise the same place."""
|
||||
if not frozen or "cells" not in frozen:
|
||||
return cited == (frozen or {})
|
||||
refs = [str(ref) for ref, _ in frozen["cells"]]
|
||||
if set(cited) != {"sheet", "cell"} or cited["sheet"] != frozen["sheet"] or not isinstance(cited["cell"], str):
|
||||
return False
|
||||
ends = cited["cell"].split(":")
|
||||
if not 1 <= len(ends) <= 2 or not all(e in refs for e in ends):
|
||||
return False
|
||||
return refs.index(ends[0]) <= refs.index(ends[-1])
|
||||
|
||||
|
||||
# Common words of each language the view can name (no word in both lists): a text's language is the one whose words
|
||||
# it uses clearly more, as a BCP 47 tag for the HTML lang attribute.
|
||||
LANGUAGE_WORDS = {
|
||||
"no": frozenset("og er det som på av til en et ikke med har den de seg var kan om ved fra også eller skal vil"
|
||||
" når så etter må blir være hvor hva dette disse andre mer bare mellom hver".split()),
|
||||
"en": frozenset("the and is of to that it not with has they was can about from also or shall will when"
|
||||
" after must be where what this these other more only between under over each are".split()),
|
||||
}
|
||||
LANGUAGE_MIN = 5 # common words a text must have before its language is told
|
||||
|
||||
|
||||
def language(text: str) -> str | None:
|
||||
"""The language of text ("no" or "en"), or None when it has too few common words or no clear winner."""
|
||||
words = re.findall(r"[^\W\d_]+", text.lower())
|
||||
counts = {lang: sum(1 for w in words if w in common) for lang, common in LANGUAGE_WORDS.items()}
|
||||
best = max(counts, key=lambda k: counts[k])
|
||||
rest = max(v for k, v in counts.items() if k != best)
|
||||
return best if counts[best] >= LANGUAGE_MIN and counts[best] >= 2 * rest else None
|
||||
661
okfwiki/cost.py
Normal file
661
okfwiki/cost.py
Normal file
|
|
@ -0,0 +1,661 @@
|
|||
"""Token cost of a v1 run from its sessions' transcripts: cost lines, stop checks and the isolation count. No model.
|
||||
|
||||
Usage:
|
||||
python3 -m okfwiki.cost harvest --transcripts DIR --okter FILE --work DIR --out kost.json [--packs FILE] [--margin M]
|
||||
python3 -m okfwiki.cost stop --lines L[,L] --cap N --unit blocks|passes|claims --done D --total T --kost FILE
|
||||
[--blocks FILE] [--batches-done B --batches-total N] [--next B [--next-chars C]]
|
||||
[--margin M]
|
||||
[--name NAME]
|
||||
python3 -m okfwiki.cost summarize --kost FILE [--cap-group NAME=L[,L]:CAP]...
|
||||
python3 -m okfwiki.cost isolation --transcripts DIR --run R --tasks DIR --out isolation.json
|
||||
[--line read|diagnose|repair]
|
||||
|
||||
harvest Reads the sessions listed in okter.txt ("<session-id> <kind>" per line, kind build, read, diagnose, merge or
|
||||
repair) under
|
||||
the transcripts directory (~/.claude/projects/<slug>/): each session's main transcript is orchestration, on
|
||||
the line orchestration-<kind>, with the input side of every turn kept by turn number, the first turn that
|
||||
starts helpers (an Agent, Task or Workflow call) and the tokens of the turns before it; every subagent
|
||||
transcript under <session>/subagents/ or, for a workflow's agents, <session>/subagents/workflows/<id>/ (the
|
||||
agent() label is the description) is attributed by its description "V1-<run>-<line>-<unit>", line pack (kept
|
||||
as build), link, read, diagnose, merge (step 18a's pair helper, order 29), repair (the repair helper, order 34)
|
||||
or calib. Any other description is an unattributed run: it is listed and the exit is 1,
|
||||
because a model-run session spawns no helper agents. Tokens are counted as the trial did: per message id
|
||||
once, at its largest value; an output never recorded is derived from the next message's input growth (a
|
||||
lower bound). With --packs, a pack log with a gap (a logged pack with no build run, or a build run for a pack
|
||||
the log lacks) cannot run. The margin is max(0.10, measured), 0.10 when nothing is measured, because the
|
||||
measurement is itself a lower bound. Recorded cap groups and stop rows survive a new harvest.
|
||||
stop The stop check after a pack or a pass. Measured = the sum of the given lines (calib is reported, never
|
||||
capped). Flat projection = measured / done x total. --unit blocks also fits cost per block against blocks
|
||||
read before (a + b*R) over the packs in --blocks (a pack log with "pack" and "blocks"), orchestration spread
|
||||
over the packs in proportion to their attributed cost, and projects a*T + b*T*T/2, once at least four packs
|
||||
are measured (the operator 11.10.2026); before that the flat projection alone. --unit claims scales only the
|
||||
helper lines per claim: an orchestration line counts the session's part before its first helper once and the
|
||||
rest per batch (a message that starts helpers, or a phase of a workflow), measured / --batches-done x
|
||||
--batches-total, both required with an orchestration line. STOP when the projection x (1 + margin) exceeds
|
||||
the cap. With --next B (--unit blocks only) the projection is recorded and printed but does not stop:
|
||||
STOP when (measured + measured / done x B) x (1 + margin) exceeds the cap, B being the blocks of the pack
|
||||
the model works on before the next check (the operator 12.10.2026: a cost line stops on measured cost plus a
|
||||
margin, the projection is reported only; order 22 session A3: plus the next pack, so the cap is not passed).
|
||||
With --next-chars C as well, the next pack's share is by characters, not blocks (motor B17: a block of a
|
||||
folder's PDF or spreadsheet holds about ten times the characters of a docs block, and the cost follows the
|
||||
characters): measured / the `block_chars` of the packs with a compiler run here x C, ROUND2_PER_CHAR before
|
||||
the first; a log whose built packs have no `block_chars` keeps the rule by blocks.
|
||||
The cap group and the stop row are recorded in the kost file.
|
||||
Exit 0 continue, 1 STOP, 2 cannot run; a caller treats every non-zero exit as STOP.
|
||||
summarize Per line "<line>: <lower bound> + margin <m> = <sum>"; per cap group (recorded by stop, or given with
|
||||
--cap-group) "<name>: <lower bound> + margin <m> = <sum> of <cap>"; orchestration tokens per turn, per pack
|
||||
and the input per turn fitted against the turn number. Exit 1 when a group is over its cap.
|
||||
isolation Reader agents (or, with --line diagnose, diagnosis agents) are one per task, described
|
||||
"V1-<run>-<line>-<task-id>", each with the task file <tasks>/<task-id>.md, started from a session or from a workflow (the same two places as harvest). Every
|
||||
Read call of another path is a break. 0 breaks over at least one agent = exit 0. With --line repair (order
|
||||
34), each repair helper "V1-<run>-repair-<task-id>" may read exactly the parts and write exactly the one
|
||||
answer path of <tasks>/<task-id>.parts.json (krepair task's part list); every other tool call, a tool other
|
||||
than Read and Write among them, is a break (CLAUDE.md: the breach count reads every tool call), and so is a
|
||||
part never read and an answer written other than once (order 39). A helper
|
||||
whose task has no part list cannot run (exit 2).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Mapping
|
||||
from fractions import Fraction
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from okfwiki.common import RunError
|
||||
from okfwiki.transcript import USAGE_FIELDS, read_transcript
|
||||
|
||||
SCHEMA = "okf-wiki/cost/1"
|
||||
ISOLATION_SCHEMA = "okf-wiki/isolation/1"
|
||||
LINES = ("build", "link", "orchestration-build", "read", "orchestration-read", "diagnose", "orchestration-diagnose",
|
||||
"merge", "orchestration-merge", "repair", "orchestration-repair", "calib")
|
||||
UNCAPPED = ("calib",)
|
||||
LINE_OF = {"pack": "build", "link": "link", "read": "read", "diagnose": "diagnose", "merge": "merge",
|
||||
"repair": "repair", "calib": "calib"}
|
||||
KINDS = ("build", "read", "diagnose", "merge", "repair")
|
||||
UNITS = ("blocks", "passes", "claims")
|
||||
ISOLATED = ("read", "diagnose", "repair")
|
||||
PREFIX = re.compile(r"^V1-(.+?)-(pack|link|read|diagnose|merge|repair|calib)-(\S+)$")
|
||||
PACK_UNIT = re.compile(r"^p(\d+)(?:-|$)")
|
||||
MARGIN_FLOOR = 0.10
|
||||
CATEGORIES = {"input_tokens": "input", "cache_read_input_tokens": "cache_read",
|
||||
"cache_creation_input_tokens": "cache_write", "output_tokens": "output"}
|
||||
INPUT_SIDE = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens")
|
||||
CHARS_PER_TOKEN_LOW = 2.5 # injected text counted high, so a derived output is a lower bound
|
||||
SPAWNS = ("Agent", "Task", "Workflow") # the tool calls that start helpers or a workflow's agents
|
||||
FIT_FROM_PACKS = 4 # the operator 11.10.2026: the fitted curve counts from pack 4
|
||||
ROUND2_PER_BLOCK = 3_259 # round 2 of the first part, 14 741 606 over 4 523 blocks (order 25 session C): the next pack's
|
||||
# share before a ledger's first compiler run (order 27 A3)
|
||||
ROUND2_PER_CHAR = Fraction(14_741_606, 1_111_796) # the same cost over the 1 111 796 characters of round 2's blocks
|
||||
# (about 13.3 per character): the share by characters before the first compiler run (B17)
|
||||
|
||||
|
||||
def margin(measured: float | None) -> float:
|
||||
"""The margin on a lower-bound measurement: the measured one, never below the 10 % floor."""
|
||||
return MARGIN_FLOOR if measured is None else max(MARGIN_FLOOR, measured)
|
||||
|
||||
|
||||
def sessions(okter: Path) -> list[tuple[str, str]]:
|
||||
"""(session id, kind) per line of okter.txt, in order, each id once."""
|
||||
out: list[tuple[str, str]] = []
|
||||
for line in okter.read_text(encoding="utf-8").splitlines():
|
||||
parts = line.split()
|
||||
if not parts:
|
||||
continue
|
||||
if len(parts) != 2 or parts[1] not in KINDS:
|
||||
raise RunError(f"okter line {line!r} is not '<session-id> <kind>' with kind in {', '.join(KINDS)}")
|
||||
known = dict(out)
|
||||
if parts[0] in known and known[parts[0]] != parts[1]:
|
||||
raise RunError(f"session {parts[0]} is listed as both {known[parts[0]]} and {parts[1]}")
|
||||
if parts[0] not in known:
|
||||
out.append((parts[0], parts[1]))
|
||||
return out
|
||||
|
||||
|
||||
def _result_chars(message: dict[str, Any]) -> int:
|
||||
content = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
return 0
|
||||
total = 0
|
||||
for item in content:
|
||||
if not (isinstance(item, dict) and item.get("type") == "tool_result"):
|
||||
continue
|
||||
inner = item.get("content")
|
||||
if isinstance(inner, str):
|
||||
total += len(inner)
|
||||
elif isinstance(inner, list):
|
||||
total += sum(len(str(x.get("text", ""))) for x in inner if isinstance(x, dict))
|
||||
return total
|
||||
|
||||
|
||||
def usage(path: Path, per_turn: bool = False) -> dict[str, Any]:
|
||||
"""Tokens of one transcript, ported from the trial. A message whose records all lack a stop reason never got its
|
||||
final usage written; when another message follows, its output is the growth of the next message's input side
|
||||
less the text injected between them (counted high at CHARS_PER_TOKEN_LOW), a lower bound. With per_turn, the
|
||||
input side of every turn is kept with its turn number."""
|
||||
per: dict[str, dict[str, int]] = {}
|
||||
order: list[str] = []
|
||||
complete: set[str] = set()
|
||||
spawning: set[str] = set()
|
||||
between: dict[str, int] = {}
|
||||
last_stop: str | None = None
|
||||
for line in read_transcript(path):
|
||||
message = line.get("message")
|
||||
if line.get("type") == "attachment" and order:
|
||||
between[order[-1]] = between.get(order[-1], 0) + len(str(line.get("rendered") or ""))
|
||||
continue
|
||||
if not isinstance(message, dict):
|
||||
continue
|
||||
if line.get("type") == "user" and order:
|
||||
between[order[-1]] = between.get(order[-1], 0) + _result_chars(message)
|
||||
continue
|
||||
if line.get("type") != "assistant":
|
||||
continue
|
||||
mid = str(message.get("id", ""))
|
||||
if mid not in per:
|
||||
order.append(mid)
|
||||
seen = per.setdefault(mid, {})
|
||||
fields = message.get("usage") or {}
|
||||
for f in USAGE_FIELDS:
|
||||
value = fields.get(f)
|
||||
if isinstance(value, int):
|
||||
seen[f] = max(seen.get(f, 0), value)
|
||||
content = message.get("content")
|
||||
if isinstance(content, list) and any(isinstance(b, dict) and b.get("type") == "tool_use"
|
||||
and b.get("name") in SPAWNS for b in content):
|
||||
spawning.add(mid)
|
||||
stop = message.get("stop_reason")
|
||||
last_stop = stop if isinstance(stop, str) else None
|
||||
if isinstance(stop, str):
|
||||
complete.add(mid)
|
||||
out: dict[str, Any] = {CATEGORIES[f]: sum(v.get(f, 0) for v in per.values()) for f in USAGE_FIELDS}
|
||||
out["output_recorded"] = out["output"]
|
||||
derived = unrecorded = 0
|
||||
for i, mid in enumerate(order):
|
||||
if mid in complete:
|
||||
continue
|
||||
if i + 1 == len(order):
|
||||
unrecorded += 1
|
||||
continue
|
||||
growth = (sum(per[order[i + 1]].get(f, 0) for f in INPUT_SIDE) - sum(per[mid].get(f, 0) for f in INPUT_SIDE)
|
||||
- int(between.get(mid, 0) / CHARS_PER_TOKEN_LOW))
|
||||
recorded = per[mid].get("output_tokens", 0)
|
||||
if growth > recorded:
|
||||
out["output"] += growth - recorded
|
||||
derived += 1
|
||||
out["output_derived_messages"] = derived
|
||||
out["output_unrecorded"] = unrecorded
|
||||
out["total"] = sum(out[c] for c in CATEGORIES.values())
|
||||
out["turns"] = len(order)
|
||||
first = per[order[0]] if order else {}
|
||||
out["startup"] = sum(first.get(f, 0) for f in INPUT_SIDE)
|
||||
out["stop_reason"] = last_stop
|
||||
if per_turn:
|
||||
out["per_turn"] = [{"turn": i, "input": sum(per[mid].get(f, 0) for f in INPUT_SIDE)}
|
||||
for i, mid in enumerate(order, 1)]
|
||||
spawn_turn = next((i for i, mid in enumerate(order, 1) if mid in spawning), None)
|
||||
before = order if spawn_turn is None else order[:spawn_turn - 1]
|
||||
out["first_helper_turn"] = spawn_turn
|
||||
out["before_first_helper"] = sum(per[mid].get(f, 0) for mid in before for f in USAGE_FIELDS)
|
||||
return out
|
||||
|
||||
|
||||
def pack_of(unit: str) -> int | None:
|
||||
m = PACK_UNIT.match(unit)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def line_totals(runs: list[dict[str, Any]], orchestration: Mapping[str, dict[str, Any]]) -> dict[str, dict[str, int]]:
|
||||
out: dict[str, dict[str, int]] = {}
|
||||
for name in LINES:
|
||||
out[name] = {c: 0 for c in (*CATEGORIES.values(), "total", "turns", "agents")}
|
||||
rows = [(r["line"], r) for r in runs] + [(f"orchestration-{o['kind']}", o) for o in orchestration.values()]
|
||||
for name, row in rows:
|
||||
for c in (*CATEGORIES.values(), "total", "turns"):
|
||||
out[name][c] += int(row[c])
|
||||
out[name]["agents"] += 1
|
||||
return out
|
||||
|
||||
|
||||
def agent_transcripts(subagents: Path, suffix: str) -> list[Path]:
|
||||
"""The agent files of one session: its own helpers and every workflow's (measured 09.10, Claude Code 2.1.295)."""
|
||||
return sorted([*subagents.glob(f"agent-*.{suffix}"), *subagents.glob(f"workflows/*/agent-*.{suffix}")])
|
||||
|
||||
|
||||
def harvest(transcripts: Path, okter: list[tuple[str, str]]) -> dict[str, Any]:
|
||||
runs: list[dict[str, Any]] = []
|
||||
unattributed: list[dict[str, Any]] = []
|
||||
orchestration: dict[str, dict[str, Any]] = {}
|
||||
for sid, kind in okter:
|
||||
top = transcripts / f"{sid}.jsonl"
|
||||
if not top.is_file(): # order 27 A3: a wrong transcripts folder would give 0 for a whole ledger without a word
|
||||
raise RunError(f"session {sid} in okter has no transcript {top}")
|
||||
orchestration[sid] = {"kind": kind, **usage(top, per_turn=True)}
|
||||
for path in agent_transcripts(transcripts / sid / "subagents", "jsonl"):
|
||||
meta_path = path.with_name(path.stem + ".meta.json")
|
||||
meta = json.loads(meta_path.read_text(encoding="utf-8")) if meta_path.is_file() else {}
|
||||
row = {"session": sid, "agent": path.stem, "agentType": meta.get("agentType"),
|
||||
"description": str(meta.get("description", "")), **usage(path)}
|
||||
m = PREFIX.match(row["description"])
|
||||
if m is None:
|
||||
unattributed.append(row)
|
||||
continue
|
||||
row["run"], row["line"], row["unit"] = m.group(1), LINE_OF[m.group(2)], m.group(3)
|
||||
row["pack"] = pack_of(m.group(3))
|
||||
runs.append(row)
|
||||
return {"sessions": [{"id": sid, "kind": kind} for sid, kind in okter], "runs": runs,
|
||||
"unattributed": unattributed, "orchestration": orchestration, "lines": line_totals(runs, orchestration)}
|
||||
|
||||
|
||||
def check_packs(runs: list[dict[str, Any]], packs_log: dict[str, Any]) -> None:
|
||||
logged = sorted(int(p["pack"]) for p in packs_log["packs"])
|
||||
if logged != list(range(1, len(logged) + 1)):
|
||||
raise RunError(f"the pack log is not packs 1..{len(logged)}: {logged}")
|
||||
built = {int(r["pack"]) for r in runs if r["line"] == "build" and r["pack"] is not None}
|
||||
missing, extra = sorted(set(logged) - built), sorted(built - set(logged))
|
||||
if missing or extra:
|
||||
raise RunError(f"pack log gap: logged without a build run {missing}, built but not logged {extra}")
|
||||
|
||||
|
||||
def allocate(pack_costs: Mapping[int, float], packs: list[list[str]], blocks_doc: dict[str, Any]) -> dict[str, float]:
|
||||
"""Tokens per source: each pack's cost split over its sources by their share of the pack's blocks."""
|
||||
count: dict[str, int] = {}
|
||||
for b in blocks_doc["blocks"]:
|
||||
count[b["file"]] = count.get(b["file"], 0) + 1
|
||||
per: dict[str, float] = {}
|
||||
for n, names in enumerate(packs, 1):
|
||||
if n not in pack_costs:
|
||||
continue
|
||||
total = sum(count.get(x, 0) for x in names) or 1
|
||||
for x in names:
|
||||
per[x] = pack_costs[n] * count.get(x, 0) / total
|
||||
return per
|
||||
|
||||
|
||||
def fit_line(points: list[tuple[float, float]]) -> tuple[float, float]:
|
||||
"""Least squares y = a + b*x; one distinct x gives b = 0."""
|
||||
xs, ys = [x for x, _ in points], [y for _, y in points]
|
||||
mx, my = sum(xs) / len(xs), sum(ys) / len(ys)
|
||||
if len(set(xs)) == 1:
|
||||
return my, 0.0
|
||||
b = sum((x - mx) * (y - my) for x, y in points) / sum((x - mx) ** 2 for x in xs)
|
||||
return my - b * mx, b
|
||||
|
||||
|
||||
def project(points: list[tuple[float, int, int]], total_blocks: int) -> dict[str, Any]:
|
||||
"""points: (cost, blocks in the unit, blocks read before it). Cost per block a + b*R, summed to a*T + b*T*T/2."""
|
||||
if not points:
|
||||
return {"fit": None, "flat": None, "T": total_blocks}
|
||||
a, b = fit_line([(float(r), c / bl) for c, bl, r in points])
|
||||
fit: dict[str, Any] = {"a": a, "b": b, "note": "one point, no growth term" if b == 0 and len(points) == 1
|
||||
else f"{len(points)} points, not validated"}
|
||||
fit["value"] = a * total_blocks + b * total_blocks * total_blocks / 2
|
||||
flat = sum(c for c, _, _ in points) / sum(bl for _, bl, _ in points) * total_blocks
|
||||
return {"fit": fit, "flat": flat, "T": total_blocks}
|
||||
|
||||
|
||||
def capped_lines(spec: str) -> list[str]:
|
||||
names = [x.strip() for x in spec.split(",") if x.strip()]
|
||||
for name in names:
|
||||
if name not in LINES:
|
||||
raise RunError(f"unknown cost line {name!r}; lines are {', '.join(LINES)}")
|
||||
if name in UNCAPPED:
|
||||
raise RunError(f"{name} is reported, never capped")
|
||||
if not names:
|
||||
raise RunError("no cost line given")
|
||||
return names
|
||||
|
||||
|
||||
def claims_projection(kost: dict[str, Any], lines: list[str], done: int, total: int,
|
||||
batches: tuple[int, int] | None) -> tuple[Fraction, dict[str, Any]]:
|
||||
"""Helper lines per claim; each orchestration line's part before its session's first helper once, the rest per
|
||||
batch. Measured 11.10 in order 12: scaling the whole sum per claim projected 162.8 M where the reading cost 10.5 M,
|
||||
because the session's fixed start was scaled with the claims."""
|
||||
helpers = sum(int(kost["lines"][name]["total"]) for name in lines if not name.startswith("orchestration-"))
|
||||
projected = Fraction(helpers) * total / done
|
||||
before = after = 0
|
||||
for name in (x for x in lines if x.startswith("orchestration-")):
|
||||
kind = name.removeprefix("orchestration-")
|
||||
line_before = sum(int(o["before_first_helper"]) for o in kost.get("orchestration", {}).values()
|
||||
if o.get("kind") == kind)
|
||||
before += line_before
|
||||
after += int(kost["lines"][name]["total"]) - line_before
|
||||
parts: dict[str, Any] = {"helpers": helpers}
|
||||
if any(x.startswith("orchestration-") for x in lines):
|
||||
if batches is None:
|
||||
raise RunError("--unit claims with an orchestration line needs --batches-done and --batches-total")
|
||||
b_done, b_total = batches
|
||||
if b_done <= 0 or b_done > b_total:
|
||||
raise ValueError(f"--batches-done {b_done} --batches-total {b_total}: need 0 < done <= total")
|
||||
projected += before + Fraction(after) * b_total / b_done
|
||||
parts.update({"before_first_helper": before, "after_first_helper": after, "batches_done": b_done,
|
||||
"batches_total": b_total})
|
||||
return projected, parts
|
||||
|
||||
|
||||
def ledger_rate(kost: dict[str, Any], packs_log: dict[str, Any], measured: int) -> tuple[Fraction, int]:
|
||||
"""(cost per block, the blocks it is over): the measured cost over the blocks of the packs with a compiler run in
|
||||
this ledger, so packs built in another ledger (a continued collection) never thin it; ROUND2_PER_BLOCK before the
|
||||
ledger's first compiler run (order 27 A3). A compiler run for a pack the log lacks cannot run."""
|
||||
blocks = {int(p["pack"]): int(p["blocks"]) for p in packs_log["packs"]}
|
||||
over = sum(blocks[n] for n in built_packs(kost, packs_log))
|
||||
return (Fraction(measured, over) if over else Fraction(ROUND2_PER_BLOCK)), over
|
||||
|
||||
|
||||
def built_packs(kost: dict[str, Any], packs_log: dict[str, Any]) -> set[int]:
|
||||
"""The packs with a compiler run in this ledger. A compiler run for a pack the log lacks cannot run."""
|
||||
logged = {int(p["pack"]) for p in packs_log["packs"]}
|
||||
built: set[int] = set()
|
||||
for r in kost.get("runs", []):
|
||||
if r.get("line") == "build" and r.get("pack") is not None:
|
||||
if int(r["pack"]) not in logged:
|
||||
raise RunError(f"a compiler run is for pack {r['pack']}, which the pack log lacks")
|
||||
built.add(int(r["pack"]))
|
||||
return built
|
||||
|
||||
|
||||
def ledger_chars_rate(kost: dict[str, Any], packs_log: dict[str, Any], measured: int) -> tuple[Fraction, int] | None:
|
||||
"""(cost per character, the characters it is over), as ledger_rate by the characters of the packs' blocks
|
||||
(`block_chars`), ROUND2_PER_CHAR before the ledger's first compiler run; None when a built pack has no
|
||||
`block_chars` (a log written before blocks had characters), which keeps the rule by blocks (B17)."""
|
||||
chars = {int(p["pack"]): p.get("block_chars") for p in packs_log["packs"]}
|
||||
built = built_packs(kost, packs_log)
|
||||
if any(not isinstance(chars[n], int) for n in built):
|
||||
return None
|
||||
over = sum(int(chars[n] or 0) for n in built)
|
||||
return (Fraction(measured, over) if over else ROUND2_PER_CHAR), over
|
||||
|
||||
|
||||
def stop_row(kost: dict[str, Any], lines: list[str], cap: int, unit: str, done: int, total: int,
|
||||
packs_log: dict[str, Any] | None, used_margin: float,
|
||||
batches: tuple[int, int] | None = None, following: int | None = None,
|
||||
next_chars: int | None = None) -> dict[str, Any]:
|
||||
if next_chars is not None and (following is None or next_chars < 0):
|
||||
raise ValueError(f"--next-chars {next_chars}: the characters of the next pack, with --next only")
|
||||
if done <= 0 or total <= 0 or done > total:
|
||||
raise ValueError(f"--done {done} --total {total}: need 0 < done <= total")
|
||||
measured = sum(int(kost["lines"][name]["total"]) for name in lines)
|
||||
flat = Fraction(measured) * total / done
|
||||
rate = Fraction(measured, done)
|
||||
parts: dict[str, Any] = {}
|
||||
if unit == "claims":
|
||||
flat, parts = claims_projection(kost, lines, done, total, batches)
|
||||
fit: float | None = None
|
||||
if unit == "blocks":
|
||||
if packs_log is None:
|
||||
raise RunError("--unit blocks needs --blocks (a pack log with pack and blocks)")
|
||||
rate, over = ledger_rate(kost, packs_log, measured)
|
||||
flat = measured + rate * (total - done) # the blocks not yet done at this ledger's rate
|
||||
parts.update(rate=float(rate), rate_blocks=over)
|
||||
attributed: dict[int, int] = {}
|
||||
for r in kost.get("runs", []):
|
||||
if r.get("line") in lines and r.get("pack") is not None:
|
||||
attributed[int(r["pack"])] = attributed.get(int(r["pack"]), 0) + int(r["total"])
|
||||
if attributed:
|
||||
scale = measured / sum(attributed.values())
|
||||
points: list[tuple[float, int, int]] = []
|
||||
read = 0
|
||||
for p in sorted(packs_log["packs"], key=lambda p: int(p["pack"])):
|
||||
n, blocks = int(p["pack"]), int(p["blocks"])
|
||||
if n in attributed:
|
||||
points.append((attributed[n] * scale, blocks, read))
|
||||
read += blocks
|
||||
fitted = project(points, total)["fit"] if len(points) >= FIT_FROM_PACKS else None
|
||||
fit = float(fitted["value"]) if fitted else None
|
||||
worst = max(flat, Fraction(fit)) if fit is not None else flat
|
||||
with_margin = worst * (1 + Fraction(str(used_margin)))
|
||||
row = {"lines": lines, "cap": cap, "unit": unit, "done": done, "total": total, "measured": measured,
|
||||
"flat": float(flat), "fit": fit, "margin": used_margin, "projected_with_margin": math.ceil(with_margin),
|
||||
"stop": with_margin > cap, **parts}
|
||||
if following is not None:
|
||||
if unit != "blocks" or following < 0:
|
||||
raise ValueError(f"--next {following}: the blocks of the next pack, with --unit blocks only")
|
||||
measured_next = (measured + rate * following) * (1 + Fraction(str(used_margin)))
|
||||
row.update(rule="measured", next=following, measured_next_with_margin=math.ceil(measured_next),
|
||||
stop=measured_next > cap)
|
||||
by_chars = ledger_chars_rate(kost, packs_log, measured) if next_chars is not None and packs_log else None
|
||||
if by_chars is not None:
|
||||
rate_c, over_c = by_chars
|
||||
measured_next = (measured + rate_c * int(next_chars or 0)) * (1 + Fraction(str(used_margin)))
|
||||
row.update(rule="measured-chars", next_chars=next_chars, rate_chars=float(rate_c), rate_chars_over=over_c,
|
||||
measured_next_with_margin=math.ceil(measured_next), stop=measured_next > cap)
|
||||
return row
|
||||
|
||||
|
||||
def with_margin(lower: int, used: float) -> int:
|
||||
return math.ceil(Fraction(lower) * (1 + Fraction(str(used))))
|
||||
|
||||
|
||||
def summarize(kost: dict[str, Any], extra_groups: Mapping[str, dict[str, Any]]) -> tuple[list[str], bool]:
|
||||
used = float(kost["margin"]["used"])
|
||||
out: list[str] = []
|
||||
for name in LINES:
|
||||
lower = int(kost["lines"].get(name, {}).get("total", 0)) # a kost file from before a line existed
|
||||
note = " (reported, never capped)" if name in UNCAPPED else ""
|
||||
out.append(f"{name}: {lower} + margin {used:.2f} = {with_margin(lower, used)}{note}")
|
||||
within = True
|
||||
for name, group in {**kost.get("caps", {}), **extra_groups}.items():
|
||||
lower = sum(int(kost["lines"][x]["total"]) for x in group["lines"])
|
||||
total = with_margin(lower, used)
|
||||
within = within and total <= int(group["cap"])
|
||||
out.append(f"{name}: {lower} + margin {used:.2f} = {total} of {group['cap']}")
|
||||
packs = {int(r["pack"]) for r in kost.get("runs", []) if r.get("pack") is not None}
|
||||
orchestration = kost.get("orchestration", {})
|
||||
for kind in KINDS:
|
||||
mine = [o for o in orchestration.values() if o.get("kind") == kind]
|
||||
turns = sum(int(o["turns"]) for o in mine)
|
||||
if not turns:
|
||||
continue
|
||||
tokens = int(kost["lines"].get(f"orchestration-{kind}", {}).get("total", 0))
|
||||
per_pack = f"; per pack {tokens // len(packs)} over {len(packs)} packs" if packs else ""
|
||||
out.append(f"orchestration-{kind} per turn: {tokens // turns} over {turns} turns{per_pack}")
|
||||
points = [(float(t["turn"]), float(t["input"])) for o in mine for t in o.get("per_turn", [])]
|
||||
if len({x for x, _ in points}) > 1:
|
||||
a, b = fit_line(points)
|
||||
out.append(f"orchestration-{kind} input per turn: {round(a)} + {round(b)} x turn ({len(points)} turns)")
|
||||
return out, within
|
||||
|
||||
|
||||
def parse_cap_group(text: str) -> tuple[str, dict[str, Any]]:
|
||||
name, sep, rest = text.partition("=")
|
||||
lines, sep2, cap = rest.rpartition(":")
|
||||
if not (sep and sep2 and name.strip()):
|
||||
raise RunError(f"--cap-group {text!r} is not NAME=LINES:CAP")
|
||||
return name.strip(), {"lines": capped_lines(lines), "cap": int(cap)}
|
||||
|
||||
|
||||
def part_list(tasks: Path, task: str) -> tuple[set[Path], Path]:
|
||||
"""(the parts a repair helper may read, the one answer path it writes) from krepair task's part list."""
|
||||
path = tasks / f"{task}.parts.json"
|
||||
if not path.is_file():
|
||||
raise RunError(f"repair task {task} has no part list {path}")
|
||||
doc = load_json(path)
|
||||
parts, answer = doc.get("parts"), doc.get("answer")
|
||||
if not (isinstance(parts, list) and parts and all(isinstance(p, str) for p in parts) and isinstance(answer, str)):
|
||||
raise RunError(f"{path} is not a part list with parts and one answer path")
|
||||
return {(tasks / p).resolve() for p in parts}, (tasks / answer).resolve()
|
||||
|
||||
|
||||
def repair_breaks(path: Path, reads: set[Path], answer: Path) -> tuple[int, int, list[str]]:
|
||||
"""(reads, writes, breaks) of one repair helper: every tool call is checked, not only Read, and a part it never
|
||||
read or an answer it wrote other than once is a break of its own (order 39)."""
|
||||
n_reads = n_writes = 0
|
||||
seen: set[Path] = set()
|
||||
breaks: list[str] = []
|
||||
for record in read_transcript(path) if path.is_file() else []:
|
||||
message = record.get("message")
|
||||
content = message.get("content") if isinstance(message, dict) else None
|
||||
for block in content if isinstance(content, list) else []:
|
||||
if not (isinstance(block, dict) and block.get("type") == "tool_use"):
|
||||
continue
|
||||
name = str(block.get("name"))
|
||||
target = (block.get("input") or {}).get("file_path")
|
||||
where = Path(str(target)).resolve() if isinstance(target, str) else None
|
||||
if name == "Read" and where in reads:
|
||||
n_reads += 1
|
||||
seen.add(where)
|
||||
elif name == "Write" and where == answer:
|
||||
n_writes += 1
|
||||
else:
|
||||
breaks.append(f"{name} {target}" if isinstance(target, str) else name)
|
||||
breaks.extend(f"part not read: {p}" for p in sorted(reads - seen))
|
||||
if n_writes != 1:
|
||||
breaks.append("answer not written" if n_writes == 0 else f"answer written {n_writes} times")
|
||||
return n_reads, n_writes, breaks
|
||||
|
||||
|
||||
def isolation(transcripts: Path, run: str, tasks: Path, line: str = "read") -> dict[str, Any]:
|
||||
prefix = re.compile(rf"^V1-{re.escape(run)}-{re.escape(line)}-(\S+)$")
|
||||
per_agent: list[dict[str, Any]] = []
|
||||
for meta_path in sorted(p for d in transcripts.glob("*/subagents") for p in agent_transcripts(d, "meta.json")):
|
||||
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
m = prefix.match(str(meta.get("description", "")))
|
||||
if m is None:
|
||||
continue
|
||||
path = meta_path.with_name(meta_path.name.removesuffix(".meta.json") + ".jsonl")
|
||||
if line == "repair":
|
||||
n_reads, n_writes, found = repair_breaks(path, *part_list(tasks, m.group(1)))
|
||||
per_agent.append({"agent": meta_path.name.removesuffix(".meta.json"), "claim": m.group(1),
|
||||
"reads": n_reads, "writes": n_writes, "breaks": found})
|
||||
continue
|
||||
task = (tasks / f"{m.group(1)}.md").resolve()
|
||||
reads: dict[str, str] = {}
|
||||
for record in read_transcript(path) if path.is_file() else []:
|
||||
message = record.get("message")
|
||||
content = message.get("content") if isinstance(message, dict) else None
|
||||
for block in content if isinstance(content, list) else []:
|
||||
if isinstance(block, dict) and block.get("type") == "tool_use" and block.get("name") == "Read":
|
||||
target = str((block.get("input") or {}).get("file_path", ""))
|
||||
reads[str(block.get("id") or len(reads))] = target
|
||||
breaks = [t for t in reads.values() if Path(t).resolve() != task]
|
||||
per_agent.append({"agent": meta_path.name.removesuffix(".meta.json"), "claim": m.group(1),
|
||||
"reads": len(reads), "breaks": breaks})
|
||||
return {"schema": ISOLATION_SCHEMA, "run": run, "line": line, "tasks": str(tasks), "agents": len(per_agent),
|
||||
"breaks": sum(len(a["breaks"]) for a in per_agent), "per_agent": per_agent}
|
||||
|
||||
|
||||
def load_json(path: Path) -> Any:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def write_json(path: Path, doc: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(doc, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
h = sub.add_parser("harvest")
|
||||
h.add_argument("--transcripts", required=True)
|
||||
h.add_argument("--okter", required=True)
|
||||
h.add_argument("--work", required=True)
|
||||
h.add_argument("--out", required=True)
|
||||
h.add_argument("--packs")
|
||||
h.add_argument("--margin", type=float)
|
||||
s = sub.add_parser("stop")
|
||||
s.add_argument("--lines", required=True)
|
||||
s.add_argument("--cap", type=int, required=True)
|
||||
s.add_argument("--unit", choices=UNITS, required=True)
|
||||
s.add_argument("--done", type=int, required=True)
|
||||
s.add_argument("--total", type=int, required=True)
|
||||
s.add_argument("--kost", required=True)
|
||||
s.add_argument("--blocks")
|
||||
s.add_argument("--batches-done", type=int)
|
||||
s.add_argument("--batches-total", type=int)
|
||||
s.add_argument("--next", type=int)
|
||||
s.add_argument("--next-chars", type=int)
|
||||
s.add_argument("--margin", type=float)
|
||||
s.add_argument("--name")
|
||||
m = sub.add_parser("summarize")
|
||||
m.add_argument("--kost", required=True)
|
||||
m.add_argument("--cap-group", action="append", default=[])
|
||||
i = sub.add_parser("isolation")
|
||||
i.add_argument("--transcripts", required=True)
|
||||
i.add_argument("--run", required=True)
|
||||
i.add_argument("--tasks", required=True)
|
||||
i.add_argument("--out", required=True)
|
||||
i.add_argument("--line", choices=ISOLATED, default="read")
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
if args.command == "harvest":
|
||||
out = Path(args.out)
|
||||
h = harvest(Path(args.transcripts), sessions(Path(args.okter)))
|
||||
if args.packs:
|
||||
check_packs(h["runs"], load_json(Path(args.packs)))
|
||||
old = load_json(out) if out.is_file() else {}
|
||||
doc = {"schema": SCHEMA, "work": str(args.work), **h,
|
||||
"margin": {"measured": args.margin, "used": margin(args.margin)},
|
||||
"caps": old.get("caps", {}), "stops": old.get("stops", {})}
|
||||
write_json(out, doc)
|
||||
print(f"harvest: {len(h['runs'])} attributed runs, {len(h['unattributed'])} unattributed, "
|
||||
f"{len(h['orchestration'])} orchestrating sessions -> {out}")
|
||||
for u in h["unattributed"]:
|
||||
print(f" unattributed: {u['agent']} {u['description']!r}")
|
||||
return 1 if h["unattributed"] else 0
|
||||
if args.command == "stop":
|
||||
kost_path = Path(args.kost)
|
||||
kost = load_json(kost_path)
|
||||
lines = capped_lines(args.lines)
|
||||
used = margin(args.margin) if args.margin is not None else float(kost["margin"]["used"])
|
||||
packs_log = load_json(Path(args.blocks)) if args.blocks else None
|
||||
given = (args.batches_done, args.batches_total)
|
||||
if (given[0] is None) != (given[1] is None):
|
||||
raise RunError("--batches-done and --batches-total go together")
|
||||
batches = None if given[0] is None else (int(given[0]), int(given[1]))
|
||||
row = stop_row(kost, lines, args.cap, args.unit, args.done, args.total, packs_log, used, batches, args.next,
|
||||
args.next_chars)
|
||||
name = args.name or ",".join(lines)
|
||||
kost.setdefault("caps", {})[name] = {"lines": lines, "cap": args.cap}
|
||||
kost.setdefault("stops", {})[name] = row
|
||||
write_json(kost_path, kost)
|
||||
fit = f", fit {row['fit']:.0f}" if row["fit"] is not None else ""
|
||||
if "batches_total" in row:
|
||||
fit += (f" (helpers {row['helpers']} x {args.total}/{args.done}, before the first helper "
|
||||
f"{row['before_first_helper']} once, after it {row['after_first_helper']} x "
|
||||
f"{row['batches_total']}/{row['batches_done']} batches)")
|
||||
if row.get("rule") == "measured-chars":
|
||||
rate = (f"{row['measured']}/{row['rate_chars_over']} (the characters with a compiler run here)"
|
||||
if row["rate_chars_over"] else
|
||||
f"{float(ROUND2_PER_CHAR):.2f} (round 2's price per character; no compiler run here)")
|
||||
print(f"{name}: measured {row['measured']} + next {row['next_chars']} characters x {rate}, "
|
||||
f"x (1 + {used:.2f}) = {row['measured_next_with_margin']} of {args.cap}: "
|
||||
f"{'STOP' if row['stop'] else 'continue'}; projection (reported only): flat {row['flat']:.0f}{fit}, "
|
||||
f"x (1 + {used:.2f}) = {row['projected_with_margin']}")
|
||||
elif "rule" in row:
|
||||
rate = (f"{row['measured']}/{row['rate_blocks']} (the blocks with a compiler run here)"
|
||||
if row["rate_blocks"] else f"{ROUND2_PER_BLOCK} (round 2's price per block; no compiler run here)")
|
||||
print(f"{name}: measured {row['measured']} + next {row['next']} blocks x {rate}, "
|
||||
f"x (1 + {used:.2f}) = {row['measured_next_with_margin']} of {args.cap}: "
|
||||
f"{'STOP' if row['stop'] else 'continue'}; projection (reported only): flat {row['flat']:.0f}{fit}, "
|
||||
f"x (1 + {used:.2f}) = {row['projected_with_margin']}")
|
||||
else:
|
||||
print(f"{name}: measured {row['measured']}, flat {row['flat']:.0f}{fit}, x (1 + {used:.2f}) = "
|
||||
f"{row['projected_with_margin']} of {args.cap}: {'STOP' if row['stop'] else 'continue'}")
|
||||
return 1 if row["stop"] else 0
|
||||
if args.command == "summarize":
|
||||
extra = dict(parse_cap_group(g) for g in args.cap_group)
|
||||
text, within = summarize(load_json(Path(args.kost)), extra)
|
||||
print("\n".join(text))
|
||||
return 0 if within else 1
|
||||
doc = isolation(Path(args.transcripts), args.run, Path(args.tasks), args.line)
|
||||
write_json(Path(args.out), doc)
|
||||
kind = {"read": "reader", "diagnose": "diagnosis", "repair": "repair"}[args.line]
|
||||
print(f"isolation: {doc['breaks']} breaks over {doc['agents']} {kind} agents")
|
||||
return 0 if doc["agents"] and doc["breaks"] == 0 else 1
|
||||
except (RunError, OSError, json.JSONDecodeError, KeyError, ValueError, ZeroDivisionError, TypeError) as exc:
|
||||
print(f"cost cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
573
okfwiki/kassemble.py
Normal file
573
okfwiki/kassemble.py
Normal file
|
|
@ -0,0 +1,573 @@
|
|||
"""K's assembly: concept pages from accepted fragments, by code. No model call (CLAUDE.md rule 1).
|
||||
|
||||
Usage:
|
||||
python3 -m okfwiki.kassemble build --work DIR --sources DIR --out DIR [--links FILE] [--replace] [--at ISO]
|
||||
|
||||
One page per concept that at least one accepted fragment (WORK/K-work/accepted.json) enriches; a concept without a
|
||||
fragment gets none. Docs fragments form the body in the order of their sources (pack, position, then line); the
|
||||
fragments of a history source go under "## History": release notes in version order, then the pages K1 marks as
|
||||
history (main_topic false, the weekly What's New pages) in week order. Every sentence ends in a footnote to its
|
||||
fragment's source entry and, for a history source, to its label entry: for a release note the entry whose quote is the
|
||||
bare version on the note's heading line, for a weekly page the entry titled with the week from the file name
|
||||
(YYYY-wNN), whose quote is the page's first heading. A fragment's source entry quotes the whole line its quote
|
||||
stands on, trimmed (`x-quote`), so the reader sees all the line says, and each context quote kpack placed (a
|
||||
heading above it or another line of its section) is a source entry of its own, cited after it. Each fragment's
|
||||
provenance mark is `x-provenance` on its source entries. SOURCES' K1 manifest (source-manifest.json) says which source is history; without it, release-* files are. A footnote label is keyed by its fragment (file,
|
||||
span, quote line, quote, provenance), never by its place on the page, so one source's fragments can be replaced
|
||||
without renumbering the others' (SPEC 5.1: "a stable `id` survives reordering").
|
||||
|
||||
A page's description is the first sentence of its first fragment, of the fragments that are its own when kmerge has
|
||||
moved others in (a fragment marked `merged_from`), so the description follows the page's name. The catalogue's other
|
||||
names for the concept (the aliases of its entry that are not the title, once, case set aside) stand on the page as
|
||||
`x-aliases`, left out when there is none, and end its index line as " (also: A; B)".
|
||||
|
||||
Main pages and homes (accepted.json main_pages and homes, written by kpack from the compiler's and the guard's
|
||||
answers): a page named as the main page of a docs source it cites carries `x-main-for`, the sorted list of those
|
||||
sources as "/" + their K1 path, and the key is left out on every other page (contract K2). A main page lists the
|
||||
pages whose home it is under "## Related concepts", one inline relative link per item, before the footnote
|
||||
definitions; a home on a page that is no main page, or on itself, places nothing.
|
||||
|
||||
Code makes no other link: every inline link in a fragment's text, to a page or to the web, is reduced to its text, and
|
||||
in-text links come only from --links, a links file of the link pass, placed by klink.place. A link, in-text or
|
||||
related, whose target page no longer exists is dropped, and the pages that held one are listed; a record of the
|
||||
--links file into or on a page that is gone is taken out of that file once the build is in (its accepted count
|
||||
follows), so no later step places it again and it never counts as stale. `generated.at` is the content's last meaningful change (SPEC 5.2): a page
|
||||
whose content is the same as in the existing output keeps its old `generated` and its bytes; any other page gets --at
|
||||
(default now), written in UTC. index.md lists every page; log.md keeps every old line and gains one entry, newest
|
||||
first (format choice d). Format choices (a)-(h) of PLAN.md section Rammer: sources[].resource starts with /, body links
|
||||
are relative, no `verified` is written, quote and locator are x- fields.
|
||||
|
||||
The output is written to a staging folder beside --out and swapped in; the old --out is set aside beside it until the
|
||||
new one is in, and comes back if the swap fails. An existing --out is refused unless --replace, and refused even then
|
||||
when it holds anything this command does not write: a file other than index.md, log.md and pages whose generated.by is
|
||||
this generator (a file browser's .DS_Store is tolerated and dropped). A marker left in WORK/K-work by an unfinished
|
||||
kpack write stops the build. Exit 0 = built, 1 = built but some link records are stale (their sentence no longer
|
||||
stands), 2 = could not run (also for an input the code did not foresee, with its traceback).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from okfwiki import audit, kblocks, klink
|
||||
from okfwiki.common import RunError, place_at, place_words
|
||||
from okfwiki.kblocks import headings_of, kilde_of, read_lines, version_key
|
||||
from okfwiki.kpack import RESERVED_IDS, Fact, check_pending, read_index, read_json, source_facts, week_of
|
||||
|
||||
GENERATOR = "okf-wiki/kassemble"
|
||||
LABEL_HEX = 10
|
||||
SAFE_ID = re.compile(r"[a-z0-9][a-z0-9-]*")
|
||||
RECORD_KEYS = {"pack": int, "position": int, "file": str, "start": int, "end": int, "section": str, "text": str,
|
||||
"quote": str, "quote_line": int, "provenance": str, "concept_id": str}
|
||||
LINK_KEYS = {"page": str, "sentence": str, "anchor": str, "at": int, "target": str}
|
||||
MAIN_KEYS = {"source": str, "concept_id": str}
|
||||
HOME_KEYS = {"source": str, "concept_id": str, "home": str}
|
||||
RELATED = "## Related concepts"
|
||||
TOLERATED = frozenset({".DS_Store"}) # folder metadata a file browser writes; dropped by the swap
|
||||
|
||||
|
||||
@dataclass
|
||||
class Page:
|
||||
name: str
|
||||
fm: dict[str, Any] # every key but `generated`
|
||||
body: str # everything after the frontmatter block
|
||||
|
||||
|
||||
@dataclass
|
||||
class Report:
|
||||
pages: int = 0
|
||||
fragments: int = 0
|
||||
new: list[str] = field(default_factory=list)
|
||||
changed: list[str] = field(default_factory=list)
|
||||
unchanged: list[str] = field(default_factory=list)
|
||||
gone: list[str] = field(default_factory=list)
|
||||
placed: int = 0
|
||||
stale: int = 0
|
||||
main_pages: int = 0
|
||||
related: int = 0
|
||||
dropped: dict[str, set[str]] = field(default_factory=dict) # page -> targets that are gone
|
||||
unlinked: int = 0 # link records into a page that is gone, taken out of the links file
|
||||
|
||||
|
||||
def frontmatter(data: dict[str, Any]) -> str:
|
||||
"""A frontmatter block; PyYAML quotes any string that would load as another type (no, 1.10)."""
|
||||
return "---\n" + yaml.safe_dump(data, sort_keys=False, allow_unicode=True, width=10_000) + "---\n"
|
||||
|
||||
|
||||
def head(fm: dict[str, Any], at: str) -> str:
|
||||
return frontmatter({**fm, "generated": {"by": GENERATOR, "at": at}})
|
||||
|
||||
|
||||
def clean(text: str) -> str:
|
||||
"""A fragment's text as body prose: no link (bundle or external) and no footnote mark of its own, on one line."""
|
||||
text = audit.FOOTNOTE_REF.sub("", audit.sub_links(lambda m: m.group(1), text))
|
||||
return re.sub(r"\s*\n\s*", " ", text).strip()
|
||||
|
||||
|
||||
def footnoted(text: str, labels: list[str]) -> str:
|
||||
if text and text[-1] not in ".!?":
|
||||
text += "."
|
||||
marks = "".join(f"[^{x}]" for x in labels)
|
||||
out, last = [], 0
|
||||
for _a, b in audit.sentence_spans(text):
|
||||
out.append(text[last:b] + marks)
|
||||
last = b
|
||||
out.append(text[last:])
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def label(r: dict[str, Any]) -> str:
|
||||
key = "\0".join([r["file"], str(r["start"]), str(r["end"]), str(r["quote_line"]), r["quote"], r["provenance"]])
|
||||
return "s" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:LABEL_HEX]
|
||||
|
||||
|
||||
def context_label(file: str, line: int, provenance: str) -> str:
|
||||
"""A context entry's label, keyed by its line and mark: fragments citing the same line share one entry."""
|
||||
key = "\0".join(["context", file, str(line), provenance])
|
||||
return "c" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:LABEL_HEX]
|
||||
|
||||
|
||||
def version_label(version: str) -> str:
|
||||
return "v" + version.replace(".", "-")
|
||||
|
||||
|
||||
def version_line(lines: list[str], version: str, file: str) -> int:
|
||||
"""The line that states the version: its heading, or the label of the component it is (kblocks.stated_versions)."""
|
||||
n = kblocks.version_line(lines, version)
|
||||
if n is None:
|
||||
raise RunError(f"{file} has no heading or label for version {version}")
|
||||
return n
|
||||
|
||||
|
||||
def shown(line: str) -> str:
|
||||
"""What a source entry quotes from a line: the whole line, trimmed, or a heading's text (order 22: a quote that is
|
||||
a piece of its line hid from the reader what the claim drew on)."""
|
||||
m = audit.HEADING.match(line)
|
||||
return (m.group(2) or "").strip() if m else line.strip()
|
||||
|
||||
|
||||
def one_line(text: str) -> str:
|
||||
return " ".join(str(text).split())
|
||||
|
||||
|
||||
def history_entry(file: str, lines: list[str], resource: str) -> dict[str, Any]:
|
||||
"""The label entry of a history page without a version: titled with the week from its file name (or, failing
|
||||
that, its first heading), quoting its first heading on that line."""
|
||||
heads = headings_of(lines)
|
||||
if not heads:
|
||||
raise RunError(f"{file} is history but has no heading to quote for its label")
|
||||
line, _level, text = heads[0]
|
||||
week = week_of(file)
|
||||
return {"id": "w" + week if week else "h-" + re.sub(r"[^a-z0-9]+", "-", file[:-3].lower()).strip("-"),
|
||||
"resource": resource, "title": week or text, "x-locator": {"line": line}, "x-quote": text,
|
||||
"x-provenance": "documented"}
|
||||
|
||||
|
||||
def history_key(r: dict[str, Any]) -> tuple[Any, ...]:
|
||||
if r.get("version"):
|
||||
return 0, version_key(f"release-v{r['version']}.md")
|
||||
week = week_of(str(r["file"]))
|
||||
return (1, week) if week else (2, str(r["file"]))
|
||||
|
||||
|
||||
def check_roles(raw: Any, key: str, keys: Mapping[str, type]) -> list[dict[str, Any]]:
|
||||
"""accepted.json's main_pages or homes: absent is empty; a record without its keys cannot run."""
|
||||
value = raw.get(key, []) if isinstance(raw, dict) else []
|
||||
if not isinstance(value, list) or not all(isinstance(r, dict) and all(type(r.get(k)) is t for k, t in keys.items())
|
||||
for r in value):
|
||||
raise RunError(f"accepted.json: {key} is not a list of records with {', '.join(keys)}")
|
||||
return value
|
||||
|
||||
|
||||
def check_records(records: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(records, list):
|
||||
raise RunError("accepted.json has no fragments list")
|
||||
for i, r in enumerate(records):
|
||||
bad = [k for k, t in RECORD_KEYS.items() if not isinstance(r, dict) or type(r.get(k)) is not t]
|
||||
if bad or not r["quote"].strip() or not (r.get("version") is None or isinstance(r["version"], str)):
|
||||
raise RunError(f"accepted record {i} lacks {bad or ['a quote or a version string']}; every fragment "
|
||||
f"carries a quote and a provenance mark (rule E)")
|
||||
context = r.get("context", [])
|
||||
if not (isinstance(context, list) and all(
|
||||
isinstance(c, dict) and type(c.get("line")) is int and isinstance(c.get("quote"), str)
|
||||
and c["quote"].strip() and isinstance(c.get("section"), str) for c in context)):
|
||||
raise RunError(f"accepted record {i}: context is not a list of records with line, quote and section")
|
||||
return records
|
||||
|
||||
|
||||
def page_models(index: dict[str, Any], records: list[dict[str, Any]], kilde: Path,
|
||||
facts: dict[str, Fact] | None = None, main_pages: list[dict[str, Any]] | None = None) -> dict[str, Page]:
|
||||
facts = facts or {}
|
||||
entries = {str(e["id"]): e for e in index["entries"]}
|
||||
by_concept: dict[str, list[dict[str, Any]]] = {}
|
||||
for r in records:
|
||||
by_concept.setdefault(r["concept_id"], []).append(r)
|
||||
cache: dict[str, list[str]] = {}
|
||||
pages: dict[str, Page] = {}
|
||||
for cid, items in sorted(by_concept.items()):
|
||||
if cid not in entries:
|
||||
raise RunError(f"concept {cid!r} has accepted fragments but no index entry")
|
||||
if cid in RESERVED_IDS or not SAFE_ID.fullmatch(cid):
|
||||
raise RunError(f"concept id {cid!r} cannot be a page name")
|
||||
e = entries[cid]
|
||||
if not (isinstance(e.get("title"), str) and e["title"].strip()):
|
||||
raise RunError(f"the index entry of {cid!r} has no title")
|
||||
title = one_line(e["title"])
|
||||
items = sorted(items, key=lambda r: (r["pack"], r["position"], r["start"], r["end"]))
|
||||
is_history = [bool(r.get("version")) or (r["file"] in facts and facts[r["file"]].history) for r in items]
|
||||
body_items = [r for r, h in zip(items, is_history) if not h]
|
||||
history = sorted((r for r, h in zip(items, is_history) if h), key=history_key)
|
||||
sources: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def add(entry: dict[str, Any]) -> str:
|
||||
known = sources.setdefault(entry["id"], entry)
|
||||
if known != entry:
|
||||
raise RunError(f"two source entries on {cid}.md share the label {entry['id']}")
|
||||
return str(entry["id"])
|
||||
|
||||
def source_lines(file: str) -> list[str]:
|
||||
if file not in cache:
|
||||
cache[file] = read_lines(kilde / file)
|
||||
return cache[file]
|
||||
|
||||
def locator(file: str, line: int, text: str, quote: str) -> dict[str, Any]:
|
||||
"""The line in the frozen copy, and for a PDF or Office source the place in the original (K1 places): in a
|
||||
spreadsheet row, the cell or cells the quote covers (common.place_at); "" for a quote names the whole row."""
|
||||
return {"line": line, **(place_at(facts[file].place(line), text, quote) if file in facts else {})}
|
||||
|
||||
def footnote(s: dict[str, Any]) -> str:
|
||||
file = PurePosixPath(s["resource"]).name
|
||||
original = facts[file].original if file in facts else None
|
||||
return f"{PurePosixPath(original).name if original else file}, {place_words(s['x-locator'])}"
|
||||
|
||||
def cite(r: dict[str, Any]) -> list[str]:
|
||||
src: dict[str, Any] = {"id": label(r), "resource": f"/kilde/{r['file']}"}
|
||||
if r["section"]:
|
||||
src["title"] = r["section"]
|
||||
lines = source_lines(r["file"])
|
||||
line = lines[r["quote_line"] - 1] if 0 < r["quote_line"] <= len(lines) else ""
|
||||
quote = shown(line) if r["quote"] in line else r["quote"]
|
||||
src.update({"x-locator": locator(r["file"], r["quote_line"], line, r["quote"]), "x-quote": quote, "x-span": [r["start"], r["end"]],
|
||||
"x-provenance": r["provenance"]})
|
||||
labels = [add(src)]
|
||||
for c in r.get("context", []):
|
||||
at = lines[c["line"] - 1] if 0 < c["line"] <= len(lines) else ""
|
||||
ctx: dict[str, Any] = {"id": context_label(r["file"], c["line"], r["provenance"]),
|
||||
"resource": f"/kilde/{r['file']}"}
|
||||
if c["section"]:
|
||||
ctx["title"] = c["section"]
|
||||
ctx.update({"x-locator": locator(r["file"], c["line"], at, ""), "x-quote": shown(at) if c["quote"] in at else c["quote"],
|
||||
"x-provenance": r["provenance"]})
|
||||
labels.append(add(ctx))
|
||||
if r.get("version"):
|
||||
v = str(r["version"])
|
||||
labels.append(add({"id": version_label(v), "resource": f"/kilde/{r['file']}", "title": v,
|
||||
"x-locator": {"line": version_line(lines, v, r["file"])},
|
||||
"x-quote": v, "x-provenance": "documented"}))
|
||||
elif r["file"] in facts and facts[r["file"]].history:
|
||||
labels.append(add(history_entry(r["file"], lines, f"/kilde/{r['file']}")))
|
||||
return labels
|
||||
|
||||
lines = [f"# {title}", ""]
|
||||
for r in body_items:
|
||||
lines += [footnoted(clean(r["text"]), cite(r)), ""]
|
||||
if history:
|
||||
lines += ["## History", ""]
|
||||
lines += [f"- {footnoted(clean(r['text']), cite(r))}" for r in history]
|
||||
lines.append("")
|
||||
lines += [f"[^{s['id']}]: {footnote(s)}" for s in sources.values()]
|
||||
own = [r for r in [*body_items, *history] if not r.get("merged_from")] # the description follows the name
|
||||
first = clean((own or body_items or history)[0]["text"])
|
||||
spans = audit.sentence_spans(first)
|
||||
fm: dict[str, Any] = {"type": "Concept", "title": title,
|
||||
"description": first[spans[0][0]:spans[0][1]] if spans else title,
|
||||
"sources": list(sources.values())}
|
||||
names = other_names(title, e.get("aliases"))
|
||||
if names:
|
||||
fm["x-aliases"] = names
|
||||
if e.get("kind") == "merge-candidate" and e.get("merge_of"):
|
||||
fm["x-merge-candidate"] = str(e["merge_of"])
|
||||
cited = {r["file"] for r in items}
|
||||
mains = sorted({f"/kilde/{m['source']}" for m in main_pages or [] if m["concept_id"] == cid
|
||||
and m["source"] in cited and m["source"] in facts and facts[m["source"]].main_topic})
|
||||
if mains:
|
||||
fm["x-main-for"] = mains
|
||||
pages[f"{cid}.md"] = Page(f"{cid}.md", fm, "\n" + "\n".join(lines) + "\n")
|
||||
return pages
|
||||
|
||||
|
||||
def other_names(title: str, aliases: Any) -> list[str]:
|
||||
"""The catalogue's other names for a concept, in its order: each alias that is not the title, once, with case set
|
||||
aside."""
|
||||
seen = {title.casefold()}
|
||||
out: list[str] = []
|
||||
for a in aliases if isinstance(aliases, list) else []:
|
||||
name = one_line(str(a))
|
||||
if name and name.casefold() not in seen:
|
||||
seen.add(name.casefold())
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
|
||||
def link_text(title: str) -> str:
|
||||
return str(title).replace("[", "(").replace("]", ")")
|
||||
|
||||
|
||||
def related(pages: dict[str, Page], homes: list[dict[str, Any]]) -> dict[str, list[str]]:
|
||||
"""Per main page, the list items of the pages whose home it is: the home links the model chose, nothing else."""
|
||||
out: dict[str, list[str]] = {}
|
||||
for name, page in pages.items():
|
||||
if not page.fm.get("x-main-for"):
|
||||
continue
|
||||
cid = name[:-3]
|
||||
held = {str(h["concept_id"]) for h in homes if h["home"] == cid and h["concept_id"] != cid
|
||||
and f"{h['concept_id']}.md" in pages}
|
||||
items = sorted(held, key=lambda c: (str(pages[f"{c}.md"].fm["title"]).casefold(), c))
|
||||
if items:
|
||||
out[name] = [f"- [{link_text(pages[f'{c}.md'].fm['title'])}]({c}.md)" for c in items]
|
||||
return out
|
||||
|
||||
|
||||
def with_related(body: str, items: list[str]) -> str:
|
||||
"""The body with the related-concepts list placed before its footnote definitions."""
|
||||
if not items:
|
||||
return body
|
||||
lines = body.split("\n")
|
||||
at = next((i for i, line in enumerate(lines) if audit.FOOTNOTE_DEF.match(line)), len(lines))
|
||||
return "\n".join(lines[:at] + [RELATED, "", *items, ""] + lines[at:])
|
||||
|
||||
|
||||
def gone_targets(old: Path, names: set[str], gone: set[str]) -> set[str]:
|
||||
"""The pages gone since the last build that the old page `old` linked to."""
|
||||
doc = audit.read_document(old, old.name)
|
||||
targets = {PurePosixPath(t.split("#", 1)[0]).name for _, t in audit.body_links(doc) if audit.is_bundle_link(t)}
|
||||
return targets & gone - names
|
||||
|
||||
|
||||
def index_text(pages: dict[str, Page]) -> str:
|
||||
"""The bundle-root index: one entry per page with its frontmatter description (SPEC 8), and after it the page's
|
||||
other names (`x-aliases`) as " (also: A; B)", so a search of the index finds a concept by any of its names."""
|
||||
out = ["---", 'okf_version: "0.2"', "---", "", "# Concepts", ""]
|
||||
out += [f"- [{link_text(p.fm['title'])}]({name}) - {link_text(p.fm['description'])}"
|
||||
+ (f" (also: {'; '.join(link_text(a) for a in p.fm['x-aliases'])})" if p.fm.get("x-aliases") else "")
|
||||
for name, p in sorted(pages.items())]
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
def log_text(old: str | None, at: str, entry: str) -> str:
|
||||
"""log.md with one new entry, newest first under its date (SPEC 9); every old line stays as it was."""
|
||||
day = at[:10]
|
||||
if old is None:
|
||||
return f"# Log\n\n## {day}\n{entry}\n"
|
||||
lines = old.split("\n")
|
||||
for i, raw in enumerate(lines):
|
||||
m = audit.HEADING.match(raw)
|
||||
text = (m.group(2) or "").strip() if m else ""
|
||||
if m and len(m.group(1)) >= 2 and audit.DATE.match(text):
|
||||
if text > day:
|
||||
raise RunError(f"the build time {at} is older than the log's newest date {text}")
|
||||
if text == day:
|
||||
lines.insert(i + 1, entry)
|
||||
else:
|
||||
lines[i:i] = [f"## {day}", entry, ""]
|
||||
return "\n".join(lines)
|
||||
return old.rstrip("\n") + f"\n\n## {day}\n{entry}\n"
|
||||
|
||||
|
||||
def generated_at(path: Path) -> str | None:
|
||||
fm = audit.read_document(path, path.name).frontmatter or {}
|
||||
gen = fm.get("generated")
|
||||
at = gen.get("at") if isinstance(gen, dict) else None
|
||||
return at if isinstance(at, str) else None
|
||||
|
||||
|
||||
def written_here(path: Path) -> bool:
|
||||
"""A concept page this command wrote: its generated.by names the generator."""
|
||||
fm = audit.read_document(path, path.name).frontmatter or {}
|
||||
gen = fm.get("generated")
|
||||
return isinstance(gen, dict) and gen.get("by") == GENERATOR
|
||||
|
||||
|
||||
def aside(out: Path) -> Path:
|
||||
return out.with_name(f".{out.name}-old")
|
||||
|
||||
|
||||
def check_out(out: Path, replace: bool) -> None:
|
||||
if aside(out).exists():
|
||||
raise RunError(f"{aside(out)} exists: an earlier swap did not finish; look at it before building again")
|
||||
if not out.exists():
|
||||
return
|
||||
if not replace:
|
||||
raise RunError(f"{out} exists; refusing to overwrite without --replace")
|
||||
if not out.is_dir():
|
||||
raise RunError(f"{out} is not a directory")
|
||||
foreign = sorted(p.name for p in out.iterdir() if p.name not in TOLERATED and not (
|
||||
p.is_file() and p.suffix == ".md" and (p.name in audit.RESERVED or written_here(p))))
|
||||
if foreign or not (out / "log.md").is_file():
|
||||
raise RunError(f"{out} holds what this command does not write ({foreign or 'no log.md'}); refusing to replace")
|
||||
|
||||
|
||||
def check_links(path: Path) -> dict[str, Any]:
|
||||
doc: dict[str, Any] = read_json(path)
|
||||
records = doc.get("links") if isinstance(doc, dict) else None
|
||||
if not isinstance(records, list) or not all(
|
||||
isinstance(r, dict) and all(type(r.get(k)) is t for k, t in LINK_KEYS.items()) for r in records):
|
||||
raise RunError(f"{path} is not a links file: a links list of records with {', '.join(LINK_KEYS)}")
|
||||
return doc
|
||||
|
||||
|
||||
def swap_in(new: Path, out: Path) -> None:
|
||||
"""Put new in out's place. An old out is set aside beside it first and comes back if the swap fails."""
|
||||
old = aside(out) if out.exists() else None
|
||||
if old is not None:
|
||||
out.rename(old)
|
||||
try:
|
||||
new.rename(out)
|
||||
except BaseException:
|
||||
if old is not None:
|
||||
old.rename(out)
|
||||
raise
|
||||
if old is not None:
|
||||
shutil.rmtree(old)
|
||||
|
||||
|
||||
def utc(at: str) -> str:
|
||||
if not audit.timestamp_ok(at):
|
||||
raise RunError(f"--at {at!r} is not an ISO 8601 datetime with an explicit UTC offset (SPEC 5)")
|
||||
return datetime.fromisoformat(at.replace("Z", "+00:00")).astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def build(work: Path, sources: Path, out: Path, links: Path | None = None, replace: bool = False,
|
||||
at: str | None = None) -> Report:
|
||||
check_pending(work)
|
||||
at = utc(at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"))
|
||||
kilde = kilde_of(sources)
|
||||
accepted = work / "K-work" / "accepted.json"
|
||||
if not accepted.is_file():
|
||||
raise RunError(f"{accepted} does not exist; accept a pack first")
|
||||
raw = read_json(accepted)
|
||||
records = check_records(raw.get("fragments") if isinstance(raw, dict) else None)
|
||||
main_pages, homes = check_roles(raw, "main_pages", MAIN_KEYS), check_roles(raw, "homes", HOME_KEYS)
|
||||
facts = source_facts(sources, sorted({str(r["file"]) for r in records} | {str(m["source"]) for m in main_pages}))
|
||||
pages = page_models(read_index(work), records, kilde, facts, main_pages)
|
||||
lists = related(pages, homes)
|
||||
check_out(out, replace)
|
||||
link_doc = check_links(links) if links is not None else None
|
||||
report = Report(pages=len(pages), fragments=len(records), main_pages=sum(bool(p.fm.get("x-main-for"))
|
||||
for p in pages.values()),
|
||||
related=sum(len(v) for v in lists.values()))
|
||||
old_pages = {p.name for p in out.glob("*.md") if klink.is_concept(p.name)} if out.is_dir() else set()
|
||||
gone = old_pages - set(pages)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
stage = Path(tempfile.mkdtemp(prefix=f".{out.name}-", dir=out.parent))
|
||||
try:
|
||||
assembled = stage / "pages"
|
||||
assembled.mkdir()
|
||||
for name, page in pages.items():
|
||||
(assembled / name).write_text(head(page.fm, at) + page.body, encoding="utf-8")
|
||||
if link_doc is not None:
|
||||
records = [{**r, "target": klink.page_rel(r["target"])} for r in link_doc["links"]]
|
||||
live = [r for r in records if r["target"] in pages and r["page"] in pages]
|
||||
for r in records:
|
||||
if r["target"] not in pages and r["page"] in pages:
|
||||
report.dropped.setdefault(r["page"], set()).add(r["target"])
|
||||
report.unlinked = len(records) - len(live) # into or on a page that is gone
|
||||
(stage / "links.json").write_text(json.dumps({**link_doc, "links": live}), encoding="utf-8")
|
||||
report.placed, report.stale = klink.place(assembled, stage / "links.json", stage / "linked", replace=False)
|
||||
assembled = stage / "linked"
|
||||
final = stage / "final"
|
||||
final.mkdir()
|
||||
for name, page in pages.items():
|
||||
_, body = klink.split_page((assembled / name).read_text(encoding="utf-8"))
|
||||
body = with_related(body, lists.get(name, []))
|
||||
old = out / name
|
||||
old_text = old.read_text(encoding="utf-8") if old.is_file() else None
|
||||
if old_text is not None:
|
||||
lost = gone_targets(old, set(pages), gone)
|
||||
if lost:
|
||||
report.dropped.setdefault(name, set()).update(lost)
|
||||
old_at = generated_at(old) if old_text is not None else None
|
||||
if old_at is not None and head(page.fm, old_at) + body == old_text:
|
||||
text = old_text
|
||||
report.unchanged.append(name)
|
||||
else:
|
||||
text = head(page.fm, at) + body
|
||||
(report.changed if old_text is not None else report.new).append(name)
|
||||
(final / name).write_text(text, encoding="utf-8")
|
||||
if out.exists():
|
||||
report.gone = sorted(p.name for p in out.glob("*.md") if klink.is_concept(p.name) and p.name not in pages)
|
||||
(final / "index.md").write_text(index_text(pages), encoding="utf-8")
|
||||
dropped = sum(len(t) for t in report.dropped.values())
|
||||
entry = (f"* **Build**: {GENERATOR}, {report.pages} pages from {report.fragments} accepted fragments: "
|
||||
f"new {len(report.new)}, changed {len(report.changed)}, unchanged {len(report.unchanged)}, "
|
||||
f"gone {len(report.gone)}" + (f"; main pages {report.main_pages}, related links {report.related}"
|
||||
if report.main_pages else "")
|
||||
+ (f"; links placed {report.placed}, stale {report.stale}" if links is not None else "")
|
||||
+ (f"; dropped links to pages that are gone {dropped}" if dropped else "") + ".")
|
||||
old_log = (out / "log.md").read_text(encoding="utf-8") if out.exists() else None
|
||||
(final / "log.md").write_text(log_text(old_log, at, entry), encoding="utf-8")
|
||||
swap_in(final, out)
|
||||
if links is not None and link_doc is not None and report.unlinked:
|
||||
kept = [r for r in link_doc["links"] if klink.page_rel(r["target"]) in pages and r["page"] in pages]
|
||||
tmp = links.with_name(f".{links.name}.tmp")
|
||||
tmp.write_text(klink.dump({**link_doc, "links": kept, "accepted": len(kept)}), encoding="utf-8")
|
||||
tmp.replace(links) # after the swap, so a build that fails keeps every record
|
||||
finally:
|
||||
shutil.rmtree(stage, ignore_errors=True)
|
||||
return report
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(prog="python3 -m okfwiki.kassemble", description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
b = sub.add_parser("build")
|
||||
b.add_argument("--work", required=True)
|
||||
b.add_argument("--sources", required=True)
|
||||
b.add_argument("--out", required=True)
|
||||
b.add_argument("--links")
|
||||
b.add_argument("--replace", action="store_true")
|
||||
b.add_argument("--at")
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
r = build(Path(args.work), Path(args.sources), Path(args.out), Path(args.links) if args.links else None,
|
||||
args.replace, args.at)
|
||||
except (RunError, OSError, UnicodeDecodeError) as exc:
|
||||
print(f"kassemble cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
except Exception: # an input this code did not foresee: "could not run", never exit 1 (stale links)
|
||||
traceback.print_exc()
|
||||
return 2
|
||||
print(f"kassemble: {r.pages} pages from {r.fragments} accepted fragments -> {args.out}")
|
||||
print(f" new {len(r.new)}, changed {len(r.changed)}, unchanged {len(r.unchanged)}, gone {len(r.gone)}")
|
||||
print(f" main pages {r.main_pages}, related links {r.related}")
|
||||
if args.links:
|
||||
print(f" links placed {r.placed}, stale {r.stale}")
|
||||
if r.dropped:
|
||||
print(f" dropped links to pages that are gone: {sum(len(t) for t in r.dropped.values())} "
|
||||
f"({', '.join(sorted(r.dropped))})")
|
||||
if r.unlinked:
|
||||
print(f" removed from the links file: {r.unlinked} records into or on pages that are gone")
|
||||
return 1 if r.stale else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
290
okfwiki/kblocks.py
Normal file
290
okfwiki/kblocks.py
Normal file
|
|
@ -0,0 +1,290 @@
|
|||
"""K's sources in a fixed order, split into blocks with line spans. No model call (CLAUDE.md rule 1).
|
||||
|
||||
Usage:
|
||||
python3 -m okfwiki.kblocks order SOURCES --out FILE
|
||||
python3 -m okfwiki.kblocks blocks SOURCES --order FILE --out FILE
|
||||
python3 -m okfwiki.kblocks total SOURCES
|
||||
|
||||
SOURCES is the folder that holds kilde/, the same raw-source root the auditor takes.
|
||||
|
||||
order every file under SOURCES/kilde/ in K's fixed order, one name per line: pages by name, then release notes
|
||||
(release-vX.Y.Z.md) by version. Any other name is a page: a docs page (docs-*), or a document of a folder
|
||||
(sources folder), .md or .txt alike.
|
||||
blocks every block of every source in the order (schema okf-wiki/blocks/1): a paragraph, one list item with its
|
||||
continuation lines, one table or one fenced code block, with its 1-based inclusive line span, its section (the
|
||||
heading path), its characters (`chars`, every line with its newline) and, for a release note, its version.
|
||||
Heading lines are section markers, not blocks, and a line that is only an HTML or MDX tag (<Note>, </Steps>)
|
||||
never starts a block, so no block is only a tag. A "#" line
|
||||
inside a component property's example string (example: `# Title ... `) is example text, not a heading.
|
||||
total the number of blocks in every file under SOURCES/kilde/.
|
||||
|
||||
Lines are counted on the text split at "\\n", as the pack and the link pass count them. Exit 0 = done, 2 = could not
|
||||
run; an existing output is never overwritten.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from okfwiki.audit import HEADING, LIST_ITEM, closes_fence, fence_marker
|
||||
from okfwiki.common import HTML_TAG_ONLY, VERSION_HEADING, RunError, write_new
|
||||
|
||||
SCHEMA = "okf-wiki/blocks/1"
|
||||
RELEASE = re.compile(r"^release-v(\d+(?:\.\d+)*)\.md$")
|
||||
VERSION_LABEL = re.compile(r'^\s*<[A-Za-z][A-Za-z0-9._-]*\s[^<>]*?\blabel="v?(\d+(?:\.\d+){2,})"')
|
||||
VERSION_BOLD = re.compile(r"^\*\*v?(\d+(?:\.\d+){2,})\*\*$")
|
||||
# A component property whose value is a backtick string left open on its line (example: `# Title): the lines after it,
|
||||
# up to the first with a backtick no backslash escapes, are example text, never a heading (order 14 findings C and D).
|
||||
STRING_OPEN = re.compile(r"^\s*[A-Za-z_$][\w$]*\s*:\s*`")
|
||||
TICK = re.compile(r"(?<!\\)`")
|
||||
# Concept test 2 of the brief (Google's reference agent), plus the generic section names of docs pages.
|
||||
META_STOP = frozenset({
|
||||
"overview", "introduction", "getting started", "quickstart", "tutorial", "walkthrough", "release notes",
|
||||
"changelog", "roadmap", "faq", "prerequisites", "next steps", "before you begin", "troubleshooting", "see also",
|
||||
"related resources", "learn more",
|
||||
})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Block:
|
||||
file: str
|
||||
start: int
|
||||
end: int
|
||||
kind: str # paragraph | item | table | code
|
||||
section: str
|
||||
version: str | None
|
||||
chars: int = 0 # the characters of its lines, each with its newline: the unit the stop line prices (B17)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Heading:
|
||||
file: str
|
||||
line: int
|
||||
level: int
|
||||
text: str
|
||||
|
||||
|
||||
def is_meta(title: str) -> bool:
|
||||
return title.strip().lower() in META_STOP
|
||||
|
||||
|
||||
def version_key(name: str) -> tuple[int, ...]:
|
||||
m = RELEASE.match(name)
|
||||
if m is None:
|
||||
raise RunError(f"{name} is not a release note name")
|
||||
return tuple(int(p) for p in m.group(1).split("."))
|
||||
|
||||
|
||||
def order(names: list[str]) -> list[str]:
|
||||
releases = [n for n in names if RELEASE.match(n)]
|
||||
return sorted(n for n in names if not RELEASE.match(n)) + sorted(releases, key=version_key)
|
||||
|
||||
|
||||
def source_files(kilde: Path) -> list[Path]:
|
||||
"""Every file under kilde, in name order: .md and .txt alike, nothing dropped by its suffix."""
|
||||
return sorted(p for p in kilde.iterdir() if p.is_file())
|
||||
|
||||
|
||||
def read_lines(path: Path) -> list[str]:
|
||||
return path.read_text(encoding="utf-8").split("\n")
|
||||
|
||||
|
||||
def string_lines(lines: list[str]) -> set[int]:
|
||||
"""The 1-based lines inside a component property's example string (STRING_OPEN), outside fenced code: from the line
|
||||
after the one that opens it to the line with the backtick that closes it, both included."""
|
||||
out: set[int] = set()
|
||||
fence = "" # the open fence's marker run, or "" outside fenced code (the auditor's rule, CommonMark 4.5)
|
||||
inside = False
|
||||
for n, raw in enumerate(lines, 1):
|
||||
if inside:
|
||||
out.add(n)
|
||||
inside = TICK.search(raw) is None
|
||||
continue
|
||||
f = fence_marker(raw)
|
||||
if fence:
|
||||
if closes_fence(f, raw, fence):
|
||||
fence = ""
|
||||
elif f:
|
||||
fence = f.group(1)
|
||||
elif STRING_OPEN.match(raw):
|
||||
inside = len(TICK.findall(raw)) % 2 == 1
|
||||
return out
|
||||
|
||||
|
||||
def headings_of(lines: list[str]) -> list[tuple[int, int, str]]:
|
||||
"""(line, level, text) of every heading outside fenced code and outside a component's example string."""
|
||||
out: list[tuple[int, int, str]] = []
|
||||
fence = ""
|
||||
text_of_string = string_lines(lines)
|
||||
for n, raw in enumerate(lines, 1):
|
||||
if n in text_of_string:
|
||||
continue
|
||||
f = fence_marker(raw)
|
||||
if fence:
|
||||
if closes_fence(f, raw, fence):
|
||||
fence = ""
|
||||
continue
|
||||
if f:
|
||||
fence = f.group(1)
|
||||
continue
|
||||
m = HEADING.match(raw)
|
||||
if m:
|
||||
out.append((n, len(m.group(1)), (m.group(2) or "").strip()))
|
||||
return out
|
||||
|
||||
|
||||
def stated_versions(lines: list[str]) -> list[tuple[int, str]]:
|
||||
"""(line, version) for every line that states a release note's version: a heading that is only the version, then
|
||||
a component whose label is the version (<Update label="X.Y.Z" ...>: every release note of the copy of 08.10, 416
|
||||
of 416, has this form and no heading; order 27 A6), then a line that is only the bold version (**X.Y.Z**: the
|
||||
page's form after 20.10, 437 of 437 on 07.11)."""
|
||||
found = [(n, m.group(1)) for n, _level, text in headings_of(lines) if (m := VERSION_HEADING.match(text))]
|
||||
found += [(n, m.group(1)) for n, raw in enumerate(lines, 1) if (m := VERSION_LABEL.match(raw))]
|
||||
return found + [(n, m.group(1)) for n, raw in enumerate(lines, 1) if (m := VERSION_BOLD.match(raw))]
|
||||
|
||||
|
||||
def version_line(lines: list[str], version: str) -> int | None:
|
||||
"""The first line that states `version`, or None."""
|
||||
return next((n for n, v in stated_versions(lines) if v == version), None)
|
||||
|
||||
|
||||
def blocks(path: Path) -> list[Block]:
|
||||
lines = read_lines(path)
|
||||
name = path.name
|
||||
version: str | None = None
|
||||
if RELEASE.match(name):
|
||||
stated = stated_versions(lines)
|
||||
version = stated[0][1] if stated else None
|
||||
out: list[Block] = []
|
||||
stack: list[tuple[int, str]] = []
|
||||
kind: str | None = None
|
||||
start = 0
|
||||
fence = ""
|
||||
text_of_string = string_lines(lines)
|
||||
|
||||
def close(end: int) -> None:
|
||||
nonlocal kind
|
||||
if kind is not None:
|
||||
out.append(Block(name, start, end, kind, " › ".join(t for _, t in stack), version,
|
||||
sum(len(line) + 1 for line in lines[start - 1:end])))
|
||||
kind = None
|
||||
|
||||
for n, raw in enumerate(lines, 1):
|
||||
f = fence_marker(raw)
|
||||
if fence:
|
||||
if closes_fence(f, raw, fence):
|
||||
fence = ""
|
||||
close(n)
|
||||
continue
|
||||
if f:
|
||||
close(n - 1)
|
||||
fence, kind, start = f.group(1), "code", n
|
||||
continue
|
||||
if not raw.strip():
|
||||
close(n - 1)
|
||||
continue
|
||||
m = None if n in text_of_string else HEADING.match(raw)
|
||||
if m:
|
||||
close(n - 1)
|
||||
level = len(m.group(1))
|
||||
stack = [(lv, t) for lv, t in stack if lv < level] + [(level, (m.group(2) or "").strip())]
|
||||
continue
|
||||
if raw.lstrip().startswith("|"):
|
||||
if kind != "table":
|
||||
close(n - 1)
|
||||
kind, start = "table", n
|
||||
continue
|
||||
if LIST_ITEM.match(raw):
|
||||
close(n - 1)
|
||||
kind, start = "item", n
|
||||
continue
|
||||
if kind in (None, "table"):
|
||||
close(n - 1)
|
||||
if HTML_TAG_ONLY.match(raw):
|
||||
continue # a tag alone is layout; the block starts at the first line of content
|
||||
kind, start = "paragraph", n
|
||||
if fence is not None:
|
||||
close(len(lines))
|
||||
else:
|
||||
last = len(lines)
|
||||
while last > 0 and not lines[last - 1].strip():
|
||||
last -= 1
|
||||
close(last)
|
||||
return out
|
||||
|
||||
|
||||
def docs_headings(kilde: Path) -> list[Heading]:
|
||||
"""The title and level 2-3 headings of every page under kilde (any file but a release note), in name order."""
|
||||
out: list[Heading] = []
|
||||
for path in (p for p in source_files(kilde) if not RELEASE.match(p.name)):
|
||||
for n, level, text in headings_of(read_lines(path)):
|
||||
if level <= 3 and text:
|
||||
out.append(Heading(path.name, n, level, text))
|
||||
return out
|
||||
|
||||
|
||||
def blocks_document(kilde: Path, names: list[str]) -> dict[str, Any]:
|
||||
found: list[dict[str, Any]] = []
|
||||
for name in names:
|
||||
path = kilde / name
|
||||
if not path.is_file():
|
||||
raise RunError(f"{path} is not a file")
|
||||
found.extend(asdict(b) for b in blocks(path))
|
||||
return {"schema": SCHEMA, "order": names, "blocks": found}
|
||||
|
||||
|
||||
def kilde_of(sources: Path) -> Path:
|
||||
kilde = sources / "kilde"
|
||||
if not kilde.is_dir():
|
||||
raise RunError(f"{kilde} is not a directory")
|
||||
return kilde
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(prog="python3 -m okfwiki.kblocks", description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
o = sub.add_parser("order")
|
||||
o.add_argument("sources")
|
||||
o.add_argument("--out", required=True)
|
||||
b = sub.add_parser("blocks")
|
||||
b.add_argument("sources")
|
||||
b.add_argument("--order", required=True)
|
||||
b.add_argument("--out", required=True)
|
||||
t = sub.add_parser("total")
|
||||
t.add_argument("sources")
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
kilde = kilde_of(Path(args.sources))
|
||||
if args.command == "order":
|
||||
fixed = order([p.name for p in source_files(kilde)])
|
||||
write_new(Path(args.out), "".join(n + "\n" for n in fixed))
|
||||
print(f"order: {len(fixed)} sources -> {args.out}")
|
||||
elif args.command == "blocks":
|
||||
names = Path(args.order).read_text(encoding="utf-8").split()
|
||||
if Path(args.out).exists():
|
||||
raise RunError(f"{args.out} exists; refusing to overwrite")
|
||||
doc = blocks_document(kilde, names)
|
||||
write_new(Path(args.out), json.dumps(doc, ensure_ascii=False, indent=1) + "\n")
|
||||
print(f"blocks: {len(doc['blocks'])} in {len(names)} sources -> {args.out}")
|
||||
else:
|
||||
files = source_files(kilde)
|
||||
print(f"total: {sum(len(blocks(p)) for p in files)} blocks in {len(files)} files of {kilde}")
|
||||
except (RunError, OSError, UnicodeDecodeError) as exc:
|
||||
print(f"kblocks cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
188
okfwiki/kguard.py
Normal file
188
okfwiki/kguard.py
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
"""The guard after a repair: the pages without a home and the sources without a main page that the auditor reports in
|
||||
a repaired collection are asked of the model again, and its answer is applied. No model call, no network, no program
|
||||
started (CLAUDE.md rule 1). Order 35 A2b and B3.3 (the operator 22.10.2026, "Lag vakten som egen kommando
|
||||
(Anbefalt)", PLAN.md § Rammer point 10): the guard of kpack run-after, used as a library and not changed.
|
||||
|
||||
Usage:
|
||||
python3 -m okfwiki.kguard ask --work DIR --bundle WIKI --sources DIR --out DIR
|
||||
python3 -m okfwiki.kguard apply --work DIR --asked FILE --answer FILE --bundle WIKI
|
||||
|
||||
- ask runs the frozen auditor in this process (kpack.run_audit) on --bundle against --sources, leaves out the retired
|
||||
checks (kpack.RETIRED) and the findings the operator approved on his list (kpack.HOMELESS_APPROVED), and reads the
|
||||
pages without a home and the sources without a main page as kpack does (klink.homeless). With no finding it writes
|
||||
nothing and exits 0. A finding the guard cannot mend (a check outside klink.GUARDED) is named and stops: exit 5, and
|
||||
nothing is written. Otherwise it writes the guard's task to --out with klink.guard_parts, the catalogue being
|
||||
kpack.main_catalogue for the last pack of packs.json, and `asked.json` (okf-wiki/guard-asked/1: the pages, the
|
||||
sources, the catalogue, the pack, the parts, the answer path, the collection's sha256, krepair.tree_sha256, and the sha256 of accepted.json), and
|
||||
exits 4: run the home task. It never writes over a file that exists.
|
||||
- apply does with the answer what kpack.apply_guard does: klink.read_guard against the pages that cite each asked
|
||||
source (klink.guard_candidates), a main page per source in the source's place, a home per page with the source that
|
||||
brings the page (kpack.first_files), then kpack.drop_dangling, written with kpack.write_state. It logs in its own
|
||||
folder, WORK/K-guard/r35-<n>/guard.json, instead of packs.json. It refuses an asked task already applied (`applied`)
|
||||
a --bundle that is not the collection asked (`stale-bundle`), and an accepted.json that changed since ask, as a
|
||||
krepair fix between ask and apply changes it without changing the bundle (`stale-accepted`): exit 1, nothing
|
||||
written.
|
||||
|
||||
Exit 0 done; 1 refused, one line per reason `refused: <code> <what>`; 2 cannot run; 4 the task is written (run it);
|
||||
5 STOP: a finding the guard cannot mend.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import traceback
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from okfwiki import klink, kpack, krepair
|
||||
from okfwiki.common import RunError
|
||||
|
||||
ASKED_SCHEMA = "okf-wiki/guard-asked/1"
|
||||
LOG_SCHEMA = "okf-wiki/guard-log/1"
|
||||
LOG_PREFIX = "r35-" # WORK/K-guard/r35-<n>/, one folder per applied answer (order 35 A2b)
|
||||
LABEL = "after the repair"
|
||||
EXIT_GUARD = kpack.EXIT_GUARD
|
||||
EXIT_BREACH = kpack.EXIT_BREACH
|
||||
|
||||
|
||||
def last_pack(work: Path) -> int:
|
||||
packs = [int(r["pack"]) for r in kpack.pack_log(work)["packs"]]
|
||||
if not packs:
|
||||
raise RunError(f"{kpack.k_work(work) / 'packs.json'} lists no pack")
|
||||
return max(packs)
|
||||
|
||||
|
||||
def accepted_sha256(work: Path) -> str:
|
||||
"""The sha256 of WORK's accepted.json, or "" when there is none: the accepted state a guard task was asked on."""
|
||||
path = kpack.k_work(work) / "accepted.json"
|
||||
return krepair.file_sha256(path) if path.is_file() else ""
|
||||
|
||||
|
||||
def cmd_ask(args: argparse.Namespace) -> int:
|
||||
work, bundle, sources, out = Path(args.work), Path(args.bundle), Path(args.sources), Path(args.out).resolve()
|
||||
kpack.check_pending(work)
|
||||
if out.exists() and any(out.iterdir()):
|
||||
raise RunError(f"{out} holds files already; the guard's task is never written over what exists")
|
||||
audit_doc = kpack.run_audit(bundle, sources)
|
||||
approved = kpack.read_approved(work)
|
||||
findings = [f for f in audit_doc["findings"] if isinstance(f, dict) and f.get("check") not in kpack.RETIRED
|
||||
and not kpack.is_approved(f, approved)]
|
||||
print(f"audit: {len(findings)} findings ("
|
||||
+ (", ".join(f"{c} {k}" for c, k in sorted(Counter(str(f.get("check")) for f in findings).items()))
|
||||
or "none") + ")")
|
||||
if not findings:
|
||||
print("guard: no page without a home, no source without a main page; no task")
|
||||
return 0
|
||||
cannot = [f for f in findings if f.get("check") not in klink.GUARDED]
|
||||
if cannot:
|
||||
print("STOP: a finding the guard cannot mend; the operator decides:")
|
||||
for f in cannot:
|
||||
print(f" {f.get('check')} {f.get('file') or ''}")
|
||||
return EXIT_BREACH
|
||||
pages, missing = klink.homeless({"findings": findings})
|
||||
last = last_pack(work)
|
||||
catalogue = [str(e["id"]) for e in kpack.main_catalogue(kpack.read_index(work), kpack.read_accepted(work), last)]
|
||||
parts = klink.guard_parts(bundle, pages, missing, catalogue, LABEL)
|
||||
paths = [out / f"part-{i}.md" for i in range(1, len(parts) + 1)]
|
||||
answer = out / "answer.json"
|
||||
asked = {"schema": ASKED_SCHEMA, "bundle": str(bundle.resolve()), "bundle_sha256": krepair.tree_sha256(bundle),
|
||||
"accepted_sha256": accepted_sha256(work), "pack": last, "pages": pages, "sources": missing, "catalogue": catalogue,
|
||||
"parts": [str(p) for p in paths], "answer": str(answer)}
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
for path, text in [*zip(paths, parts), (out / "asked.json", kpack.dump(asked))]:
|
||||
with path.open("x", encoding="utf-8") as fh:
|
||||
fh.write(text)
|
||||
print(f"guard: {len(pages)} pages without a home, {len(missing)} sources without a main page")
|
||||
for p in paths:
|
||||
print(f" part: {p}")
|
||||
print(f" answer: {answer}")
|
||||
return EXIT_GUARD
|
||||
|
||||
|
||||
def logs(work: Path) -> list[Path]:
|
||||
return sorted((work / "K-guard").glob(f"{LOG_PREFIX}*/guard.json"))
|
||||
|
||||
|
||||
def cmd_apply(args: argparse.Namespace) -> int:
|
||||
work, bundle, asked_path, answer = Path(args.work), Path(args.bundle), Path(args.asked), Path(args.answer)
|
||||
kpack.check_pending(work)
|
||||
asked: dict[str, Any] = kpack.read_json(asked_path)
|
||||
if asked.get("schema") != ASKED_SCHEMA:
|
||||
raise RunError(f"{asked_path} is not the guard's asked task ({ASKED_SCHEMA})")
|
||||
asked_sha = krepair.file_sha256(asked_path)
|
||||
refusals: list[krepair.Refusal] = []
|
||||
for log in logs(work):
|
||||
if json.loads(log.read_text(encoding="utf-8")).get("asked_sha256") == asked_sha:
|
||||
refusals.append(krepair.Refusal("applied", f"{asked_path} (logged in {log})"))
|
||||
if krepair.tree_sha256(bundle) != asked["bundle_sha256"]:
|
||||
refusals.append(krepair.Refusal("stale-bundle", f"{bundle} is not the collection {asked_path} asked"))
|
||||
if accepted_sha256(work) != asked.get("accepted_sha256"):
|
||||
refusals.append(krepair.Refusal("stale-accepted", f"{kpack.k_work(work) / 'accepted.json'} changed since "
|
||||
f"{asked_path} was asked; ask again"))
|
||||
if refusals:
|
||||
return krepair.refuse(refusals)
|
||||
last = int(asked["pack"])
|
||||
got = klink.read_guard(kpack.read_json(answer), asked["pages"], asked["sources"], asked["catalogue"],
|
||||
klink.guard_candidates(bundle, asked["sources"]))
|
||||
acc = kpack.read_accepted(work)
|
||||
for src, pid in got.main_pages:
|
||||
name = kpack.source_name(src)
|
||||
acc["main_pages"] = [m for m in acc["main_pages"] if m["source"] != name] + [
|
||||
{"source": name, "concept_id": pid, "pack": last, "by": "guard"}]
|
||||
firsts = kpack.first_files(acc)
|
||||
for pid, hid in got.homes:
|
||||
if pid not in firsts:
|
||||
raise RunError(f"the guard homed {pid}, which has no accepted fragment")
|
||||
acc["homes"].append({"source": firsts[pid], "concept_id": pid, "home": hid, "pack": last, "by": "guard"})
|
||||
dropped = kpack.drop_dangling(acc)
|
||||
answered = {"homes": len(got.homes), "main_pages": len(got.main_pages), "none": got.none,
|
||||
"unanswered": got.unanswered, "rejected": dict(sorted(got.rejected.items())),
|
||||
**kpack.nonzero(dropped=dropped[0] + dropped[1])}
|
||||
n = len(logs(work)) + 1
|
||||
folder = work / "K-guard" / f"{LOG_PREFIX}{n}"
|
||||
if folder.exists():
|
||||
raise RunError(f"{folder} exists; a guard log is never written over")
|
||||
entry = {"schema": LOG_SCHEMA, "at": krepair.now(), "asked": str(asked_path.resolve()), "asked_sha256": asked_sha,
|
||||
"answer": str(answer.resolve()), "answer_sha256": krepair.file_sha256(answer),
|
||||
"bundle_sha256": asked["bundle_sha256"], "pack": last, "answered": answered}
|
||||
kpack.write_state(work, f"guard-{LOG_PREFIX}{n}", {"accepted.json": acc},
|
||||
{folder / "guard.json": kpack.dump(entry)})
|
||||
print(f"guard answer: homes {len(got.homes)}, main pages {len(got.main_pages)}, none {len(got.none)}, "
|
||||
f"unanswered {len(got.unanswered)}, rejected {sum(got.rejected.values())} -> {folder / 'guard.json'}")
|
||||
kpack.print_dropped(*dropped)
|
||||
return 0
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(prog="python3 -m okfwiki.kguard", description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
a = sub.add_parser("ask")
|
||||
for name in ("--work", "--bundle", "--sources", "--out"):
|
||||
a.add_argument(name, required=True)
|
||||
y = sub.add_parser("apply")
|
||||
for name in ("--work", "--asked", "--answer", "--bundle"):
|
||||
y.add_argument(name, required=True)
|
||||
return p
|
||||
|
||||
|
||||
COMMANDS = {"ask": cmd_ask, "apply": cmd_apply}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
return COMMANDS[args.command](args)
|
||||
except (RunError, OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
print(f"kguard cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
except Exception: # an input this code did not foresee: still "could not run", with the traceback
|
||||
traceback.print_exc()
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
957
okfwiki/klink.py
Normal file
957
okfwiki/klink.py
Normal file
|
|
@ -0,0 +1,957 @@
|
|||
"""The link pass: the model proposes links by meaning; this code strips, asks, accepts and places them. No model call.
|
||||
|
||||
Usage:
|
||||
python3 -m okfwiki.klink strip BUNDLE --out DIR
|
||||
python3 -m okfwiki.klink sentences --accepted FILE --pack N --bundle DIR --out FILE
|
||||
python3 -m okfwiki.klink task BUNDLE --sentences FILE --catalogue IDS --out DIR
|
||||
python3 -m okfwiki.klink accept PROPOSALS --bundle DIR --links FILE
|
||||
python3 -m okfwiki.klink place BUNDLE --links FILE --out DIR [--replace]
|
||||
python3 -m okfwiki.klink pass-before --accepted FILE --pack N [--to M] --bundle DIR [--catalogue IDS] --out DIR
|
||||
python3 -m okfwiki.klink pass-after --pass-dir DIR --bundle DIR --links FILE --linked DIR --sources DIR
|
||||
--transcripts DIR --okter FILE --work DIR --kost FILE --lines L[,L] --cap N --unit U --done D --total T
|
||||
[--blocks FILE] [--name NAME]
|
||||
|
||||
Links are data (`links.json`, schema okf-wiki/links/1), never edits to pages, because K re-assembles pages from
|
||||
fragments. A record is {page, sentence, anchor, at, target}: the sentence key, the anchor text and its character
|
||||
offset in that sentence, and the target page. The sentence key is the first 16 hex characters of the sha256 of the
|
||||
sentence's original text (sliced with audit.sentence_spans from the unmasked text) after footnote marks are removed,
|
||||
every markdown link is reduced to its anchor text and whitespace is collapsed, plus "-<n>", the occurrence of that
|
||||
text on the page. Every command first reduces each bundle link in a concept body to its anchor text, in prose, code,
|
||||
headings and footnote definitions alike, so keys and offsets are computed on link-free text and placement is
|
||||
deterministic. index.md and log.md are copied unchanged.
|
||||
|
||||
strip copies the bundle with every bundle link in a concept body reduced to its anchor text.
|
||||
sentences the sentence keys on assembled pages that come from pack N's accepted fragments (`pack`, `concept_id`,
|
||||
`text`). Exit 1 when the pack has no sentence (an empty denominator).
|
||||
task renders the task parts: the contract, the catalogue as `id | title` lines and, per page, only the listed
|
||||
sentences with their keys; parts are capped at 60,000 characters. A catalogue that alone exceeds the cap
|
||||
cannot run (exit 2): it has to be split or filtered first.
|
||||
accept validates proposals {"links": [{page, sentence, anchor, target}], "none": [ids]} and appends accepted
|
||||
records; rejects are counted per reason, `none` answers are counted, and no record is ever rewritten.
|
||||
place writes every record into a copy of the bundle; refuses an existing output unless --replace (then it writes
|
||||
a temporary directory and swaps it in). A record whose sentence no longer exists is stale: exit 1.
|
||||
pass-before / pass-after chain the non-model steps of one pass (sentences -> task; accept -> place -> link rule ->
|
||||
cost harvest -> cost stop) and print only counts, so a pass costs the orchestrator few turns.
|
||||
With --to M, pass-before covers packs N..M, one directory p<n> per pack, and pass-after on the parent
|
||||
accepts every p<n> in pack order: the same links as one pass per pack, for the same orchestrator turns.
|
||||
With --unit blocks, --done is the blocks of the accepted packs the pass covers (the done line of run-after
|
||||
for its last pack), and the stop line divides by the blocks of every pack with a compiler run: in the
|
||||
skill's chain the compiler of the next pack has run before this pass-after, so the harvest holds its run
|
||||
and both sides of the stop line cover the same packs. The orchestration is one line for the whole session
|
||||
and is not split per pack. A --done that is not those blocks, or a run on a capped line that is tied to no
|
||||
pack or to a pack the log lacks, cannot run (exit 2, STOP). The stop is measured cost plus the blocks of a
|
||||
compiled pack not yet accepted (cost stop --next, and --next-chars when the pack log has their characters), and the projection is reported only (the operator
|
||||
12.10.2026).
|
||||
The catalogue of a pack is filtered to its candidate targets within CATALOGUE_CHARS (filter_catalogue)
|
||||
from a pool: the ids in --catalogue, or else the concepts with fragments in this or an earlier pack.
|
||||
Exit 0 done, 1 a counted failure (stale, empty, STOP), 2 cannot run.
|
||||
|
||||
The guard (contract K2 point 17; kpack run-after calls it, it has no command of its own): `homeless` takes the pages
|
||||
with a no-main-page-link finding and the sources with a main-page-missing finding from the auditor's JSON, so the
|
||||
auditor is the only judge. `guard_parts` renders one task for the model with GUARD_CONTRACT, the pack's filtered
|
||||
catalogue of main pages, each page without a home (id, title, description) and each source without a main page with
|
||||
the pages that cite it; `read_guard` checks the answer: a home or none per page, a main page per source. The guard's
|
||||
task runs on the linker agent, because a home is a link from a main page chosen by meaning.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import posixpath
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from okfwiki import audit, cost
|
||||
from okfwiki.common import RunError, write_new
|
||||
|
||||
SCHEMA = "okf-wiki/links/1"
|
||||
SENTENCES_SCHEMA = "okf-wiki/link-sentences/1"
|
||||
PARTS_SCHEMA = "okf-wiki/link-parts/1"
|
||||
PART_CHARS = 60_000
|
||||
CATALOGUE_CHARS = 32_000 # leaves ~27,000 characters of a part for sentences; order 28 A3: with the main pages first, the
|
||||
# fewest far concepts a pack of the rest saw in the model was 205 (24,000: three packs under G's 179)
|
||||
RECENT_PACKS = 2
|
||||
MAX_LINKS = 3
|
||||
PART_HEADER_ROOM = len("<!-- part 999 of 999 -->\n")
|
||||
FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})")
|
||||
REASONS = ("malformed", "sentence-missing", "target-missing", "self-link", "no-footnote", "anchor-not-in-sentence",
|
||||
"anchor-in-code", "duplicate", "over-limit", "overlap")
|
||||
|
||||
CONTRACT = """## Contract
|
||||
|
||||
You add links to sentences of a knowledge collection. Every sentence below has a key. Link by meaning, not by name:
|
||||
a link says that a phrase in the sentence is about a concept in the catalogue, and the phrase may use other words than
|
||||
the concept's title.
|
||||
|
||||
- Targets: only ids from the catalogue in this task. Never link a page to itself.
|
||||
- Anchor: a phrase that already stands, character for character, in the sentence, outside backticks. Change no word.
|
||||
- At most 3 links in one sentence. No lists of related pages and no "see also" lines.
|
||||
- When nothing in the catalogue fits, put the page id in `none`.
|
||||
- Output: one JSON object `{"links": [{"page": ..., "sentence": ..., "anchor": ..., "target": ...}], "none": [...]}`,
|
||||
where `page` is the page file as the task names it, `sentence` is the sentence key and `target` is a catalogue id.
|
||||
Write it to the output path named in your prompt, then stop.
|
||||
"""
|
||||
|
||||
GUARD_CONTRACT = """## Contract (home task)
|
||||
|
||||
A main page is the page for the main topic of one docs source. Every other page needs a home: a main page that links
|
||||
to it in its list of related concepts. The pages and sources below lack one. Decide by meaning, as a reader of the
|
||||
main page would look for the page.
|
||||
|
||||
- For each page under "Pages without a home": choose its home from the catalogue, or a page you name a main page
|
||||
below, and add {"page": <id>, "home": <id>} to `homes`. When no main page fits, put the page id in `none`: a page
|
||||
without a natural home is reported, never given a forced home.
|
||||
- For each source under "Sources without a main page": choose, from the pages listed under it, the page whose topic is
|
||||
that source's main topic, and add {"source": <path>, "page": <id>} to `main_pages`.
|
||||
- Output: one JSON object `{"homes": [...], "none": [...], "main_pages": [...]}`. Write it to the output path named
|
||||
in your prompt, then stop.
|
||||
"""
|
||||
GUARDED = frozenset({"no-main-page-link", "main-page-missing"})
|
||||
GUARD_REASONS = ("malformed", "page-not-asked", "duplicate", "self-home", "home-not-in-catalogue", "source-not-asked",
|
||||
"page-not-a-candidate")
|
||||
KILDE_PATH = re.compile(r"(?<![\w.-])/?(kilde/[a-z0-9._-]*[a-z0-9_-])")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Sentence:
|
||||
key: str
|
||||
text: str # the original slice of the link-reduced body, footnote marks kept
|
||||
masked: str # the same slice with inline code masked
|
||||
start: int # offset of the slice in the link-reduced body
|
||||
footnoted: bool
|
||||
labels: tuple[str, ...]
|
||||
|
||||
|
||||
# --- pages -----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def split_page(text: str) -> tuple[str, str]:
|
||||
"""(frontmatter block with its closing fence and newline, body); a page without frontmatter has an empty head."""
|
||||
lines = text.split("\n")
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return "", text
|
||||
end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
|
||||
if end is None:
|
||||
return "", text
|
||||
return "\n".join(lines[:end + 1]) + "\n", "\n".join(lines[end + 1:])
|
||||
|
||||
|
||||
def reduce_links(body: str) -> tuple[str, int]:
|
||||
"""Every bundle link reduced to its anchor text, wherever it stands in the body; external links are kept."""
|
||||
count = 0
|
||||
|
||||
def one(m: re.Match[str]) -> str:
|
||||
nonlocal count
|
||||
if not audit.is_bundle_link(m.group(2)):
|
||||
return m.group(0)
|
||||
count += 1
|
||||
return m.group(1)
|
||||
|
||||
return audit.sub_links(one, body), count
|
||||
|
||||
|
||||
def is_concept(rel: str) -> bool:
|
||||
return rel.endswith(".md") and PurePosixPath(rel).name not in audit.RESERVED
|
||||
|
||||
|
||||
def key_text(text: str) -> str:
|
||||
plain = audit.sub_links(lambda m: m.group(1), audit.FOOTNOTE_REF.sub("", text))
|
||||
return " ".join(plain.split())
|
||||
|
||||
|
||||
def prose_spans(body: str) -> list[tuple[int, int]]:
|
||||
"""(start, end) offsets of paragraphs and list items; headings, footnote definitions and fenced blocks are not prose."""
|
||||
spans: list[tuple[int, int]] = []
|
||||
current: list[int] | None = None
|
||||
fence = ""
|
||||
offset = 0
|
||||
for raw in body.split("\n"):
|
||||
start, end = offset, offset + len(raw)
|
||||
offset = end + 1
|
||||
f = FENCE.match(raw)
|
||||
if fence:
|
||||
if f and f.group(1)[0] == fence[0] and len(f.group(1)) >= len(fence):
|
||||
fence = ""
|
||||
continue
|
||||
if f:
|
||||
fence, current = f.group(1), None
|
||||
continue
|
||||
if not raw.strip() or audit.HEADING.match(raw) or audit.FOOTNOTE_DEF.match(raw.strip()):
|
||||
current = None
|
||||
continue
|
||||
item = audit.LIST_ITEM.match(raw)
|
||||
if item:
|
||||
current = [start + item.end(), end]
|
||||
spans.append((current[0], current[1]))
|
||||
elif current is None:
|
||||
current = [start + len(raw) - len(raw.lstrip()), end]
|
||||
spans.append((current[0], current[1]))
|
||||
else:
|
||||
current[1] = end
|
||||
spans[-1] = (current[0], current[1])
|
||||
return spans
|
||||
|
||||
|
||||
def body_sentences(body: str) -> list[Sentence]:
|
||||
"""The sentences of a link-reduced body, each with its key."""
|
||||
out: list[Sentence] = []
|
||||
seen: Counter[str] = Counter()
|
||||
for a, b in prose_spans(body):
|
||||
block = body[a:b]
|
||||
masked = audit.mask_code(block)
|
||||
for s, e in audit.sentence_spans(block):
|
||||
text = block[s:e]
|
||||
norm = key_text(text)
|
||||
digest = hashlib.sha256(norm.encode("utf-8")).hexdigest()[:16]
|
||||
labels = tuple(audit.FOOTNOTE_REF.findall(masked[s:e]))
|
||||
out.append(Sentence(f"{digest}-{seen[digest]}", text, masked[s:e], a + s, bool(labels), labels))
|
||||
seen[digest] += 1
|
||||
return out
|
||||
|
||||
|
||||
def page_sentences(text: str) -> list[Sentence]:
|
||||
return body_sentences(reduce_links(split_page(text)[1])[0])
|
||||
|
||||
|
||||
@dataclass
|
||||
class Page:
|
||||
rel: str
|
||||
head: str
|
||||
body: str # link-reduced
|
||||
title: str
|
||||
resources: dict[str, str] # sources[].id -> resource
|
||||
description: str = ""
|
||||
|
||||
|
||||
def read_pages(bundle: Path) -> dict[str, Page]:
|
||||
pages: dict[str, Page] = {}
|
||||
for p in sorted(bundle.rglob("*.md")):
|
||||
rel = p.relative_to(bundle).as_posix()
|
||||
if not is_concept(rel):
|
||||
continue
|
||||
doc = audit.read_document(p, rel)
|
||||
head, body = split_page(p.read_text(encoding="utf-8"))
|
||||
fm = doc.frontmatter or {}
|
||||
srcs = fm.get("sources")
|
||||
resources = {str(s["id"]): str(s.get("resource", "")) for s in (srcs if isinstance(srcs, list) else [])
|
||||
if isinstance(s, dict) and "id" in s}
|
||||
pages[rel] = Page(rel, head, reduce_links(body)[0], str(fm.get("title") or PurePosixPath(rel).stem), resources,
|
||||
" ".join(str(fm.get("description") or "").split()))
|
||||
return pages
|
||||
|
||||
|
||||
def page_rel(name: str) -> str:
|
||||
name = name.strip()
|
||||
return name if name.endswith(".md") else name + ".md"
|
||||
|
||||
|
||||
def load_json(path: Path) -> Any:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def dump(doc: Any) -> str:
|
||||
return json.dumps(doc, ensure_ascii=False, indent=1) + "\n"
|
||||
|
||||
|
||||
# --- strip and place -------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def copy_reduced(bundle: Path, out: Path, records: list[dict[str, Any]] | None) -> tuple[int, int, int]:
|
||||
"""Copy bundle to out with every bundle link reduced, then place records; (links reduced, placed, stale)."""
|
||||
by_page: dict[str, list[dict[str, Any]]] = {}
|
||||
for r in records or []:
|
||||
by_page.setdefault(str(r["page"]), []).append(r)
|
||||
reduced = placed = stale = 0
|
||||
for p in sorted(bundle.rglob("*")):
|
||||
if not p.is_file():
|
||||
continue
|
||||
rel = p.relative_to(bundle).as_posix()
|
||||
dst = out / rel
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
if not is_concept(rel):
|
||||
shutil.copyfile(p, dst)
|
||||
continue
|
||||
head, body = split_page(p.read_text(encoding="utf-8"))
|
||||
body, n = reduce_links(body)
|
||||
reduced += n
|
||||
body, ok, bad = place_records(rel, body, by_page.pop(rel, []))
|
||||
placed, stale = placed + ok, stale + bad
|
||||
dst.write_text(head + body, encoding="utf-8")
|
||||
stale += sum(len(v) for v in by_page.values()) # records for pages that no longer exist
|
||||
return reduced, placed, stale
|
||||
|
||||
|
||||
def place_records(rel: str, body: str, records: list[dict[str, Any]]) -> tuple[str, int, int]:
|
||||
by_key = {s.key: s for s in body_sentences(body)}
|
||||
edits: list[tuple[int, int, str]] = []
|
||||
stale = 0
|
||||
for r in records:
|
||||
s = by_key.get(str(r["sentence"]))
|
||||
anchor, at = str(r["anchor"]), int(r["at"])
|
||||
if s is None or s.text[at:at + len(anchor)] != anchor:
|
||||
stale += 1
|
||||
continue
|
||||
target = posixpath.relpath(str(r["target"]), posixpath.dirname(rel) or ".")
|
||||
edits.append((s.start + at, s.start + at + len(anchor), f"[{anchor}]({target})"))
|
||||
for a, b, link in sorted(edits, reverse=True):
|
||||
body = body[:a] + link + body[b:]
|
||||
return body, len(edits), stale
|
||||
|
||||
|
||||
def strip(bundle: Path, out: Path) -> int:
|
||||
if out.exists():
|
||||
raise RunError(f"{out} exists; refusing to overwrite")
|
||||
return copy_reduced(bundle, out, None)[0]
|
||||
|
||||
|
||||
def place(bundle: Path, links: Path, out: Path, replace: bool) -> tuple[int, int]:
|
||||
records = load_json(links)["links"]
|
||||
if out.exists() and not replace:
|
||||
raise RunError(f"{out} exists; refusing to overwrite without --replace")
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = Path(tempfile.mkdtemp(prefix=f".{out.name}-", dir=out.parent))
|
||||
try:
|
||||
_, placed, stale = copy_reduced(bundle, tmp, records)
|
||||
if out.exists():
|
||||
old = out.with_name(f".{out.name}-old")
|
||||
out.rename(old)
|
||||
tmp.rename(out)
|
||||
shutil.rmtree(old)
|
||||
else:
|
||||
tmp.rename(out)
|
||||
finally:
|
||||
if tmp.exists():
|
||||
shutil.rmtree(tmp)
|
||||
return placed, stale
|
||||
|
||||
|
||||
# --- what the model is asked -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def pack_sentences(accepted: dict[str, Any], pack: int, pages: dict[str, Page]) -> dict[str, list[str]]:
|
||||
texts: dict[str, list[str]] = {}
|
||||
for f in accepted["fragments"]:
|
||||
if int(f["pack"]) == pack:
|
||||
texts.setdefault(page_rel(str(f["concept_id"])), []).append(key_text(str(f["text"])))
|
||||
out: dict[str, list[str]] = {}
|
||||
for rel, frags in texts.items():
|
||||
if rel not in pages:
|
||||
continue
|
||||
keys = [s.key for s in body_sentences(pages[rel].body)
|
||||
if s.footnoted and any(key_text(s.text).rstrip(".!?") in t for t in frags)]
|
||||
if keys:
|
||||
out[rel] = keys
|
||||
return out
|
||||
|
||||
|
||||
def concept_id(rel: str) -> str:
|
||||
return PurePosixPath(rel).with_suffix("").as_posix()
|
||||
|
||||
|
||||
def pack_pool(accepted: dict[str, Any], pack: int, pages: dict[str, Page]) -> list[str]:
|
||||
"""The concepts with fragments in this or an earlier pack: the pages that exist after pack N in an additive build."""
|
||||
ids = {concept_id(page_rel(str(f["concept_id"]))) for f in accepted["fragments"] if int(f["pack"]) <= pack}
|
||||
return sorted(cid for cid in ids if page_rel(cid) in pages)
|
||||
|
||||
|
||||
def filter_catalogue(accepted: dict[str, Any], pack: int, pool: list[str], pages: dict[str, Page],
|
||||
budget: int = CATALOGUE_CHARS) -> list[str]:
|
||||
"""The candidate targets of a pack, ranked, cut at the first that does not fit the character budget.
|
||||
|
||||
The main pages (accepted.json's main_pages) come first, then the rest, each in rank order (order 28 A3: they are
|
||||
the homes of criterion 6, and with the old rule a late pack of collection 1 saw few of them). Rank: (0) the
|
||||
concepts this pack writes to; (1) those last written to in the RECENT_PACKS packs before it, latest first, because
|
||||
the fixed order puts related sources next to each other; (2) the rest by how many distinct source files their page
|
||||
cites, most first, so a late pack can still reach the central concepts made early. Ties by id. The rule uses
|
||||
structure only (packs, citations and main pages), never names: the model links by meaning.
|
||||
"""
|
||||
last: dict[str, int] = {}
|
||||
for f in accepted["fragments"]:
|
||||
p, cid = int(f["pack"]), concept_id(page_rel(str(f["concept_id"])))
|
||||
if p <= pack:
|
||||
last[cid] = max(last.get(cid, 0), p)
|
||||
|
||||
def rank(cid: str) -> tuple[int, int, str]:
|
||||
lp = last.get(cid)
|
||||
if lp == pack:
|
||||
return 0, 0, cid
|
||||
if lp is not None and pack - lp <= RECENT_PACKS:
|
||||
return 1, -lp, cid
|
||||
return 2, -len(set(pages[page_rel(cid)].resources.values()) - {""}), cid
|
||||
|
||||
mains = {concept_id(page_rel(str(m["concept_id"]))) for m in accepted.get("main_pages", [])}
|
||||
ranked = sorted(pool, key=rank)
|
||||
out: list[str] = []
|
||||
used = 0
|
||||
for cid in [c for c in ranked if c in mains] + [c for c in ranked if c not in mains]:
|
||||
size = len(catalogue_text([cid], pages))
|
||||
if used + size > budget:
|
||||
break
|
||||
out.append(cid)
|
||||
used += size
|
||||
return out
|
||||
|
||||
|
||||
@dataclass
|
||||
class Section:
|
||||
page: str
|
||||
lines: list[str]
|
||||
heading: str
|
||||
|
||||
|
||||
def catalogue_text(ids: list[str], pages: dict[str, Page]) -> str:
|
||||
out = []
|
||||
for cid in ids:
|
||||
rel = page_rel(cid)
|
||||
if rel not in pages:
|
||||
raise RunError(f"catalogue id {cid!r} is not a concept page in the bundle")
|
||||
out.append(f"{PurePosixPath(rel).with_suffix('').as_posix()} | {pages[rel].title}\n")
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def part_text(label: str, catalogue: str, sections: list[Section]) -> str:
|
||||
body = "".join(f"### {s.heading}\n" + "".join(s.lines) + "\n" for s in sections)
|
||||
return (f"# Link pass: {label}\n\n" + CONTRACT + "\n## Catalogue (link targets)\n\n" + catalogue
|
||||
+ "\n## Sentences\n\n" + body)
|
||||
|
||||
|
||||
def render_sections(label: str, fixed_ids: list[str], sections: list[Section], pages: dict[str, Page],
|
||||
part_chars: int) -> list[tuple[str, dict[str, Any]]]:
|
||||
fixed = catalogue_text(fixed_ids, pages)
|
||||
if len(fixed) > part_chars:
|
||||
raise RunError(f"the catalogue alone is {len(fixed)} characters ({len(fixed_ids)} concepts), over the part "
|
||||
f"limit of {part_chars}: split or filter the catalogue before this pass")
|
||||
room = part_chars - PART_HEADER_ROOM
|
||||
|
||||
def size(group: list[Section]) -> int:
|
||||
return len(part_text(label, fixed, group))
|
||||
|
||||
if size([]) > room:
|
||||
raise RunError(f"the contract and catalogue leave no room for a sentence within {part_chars} characters")
|
||||
chunks: list[Section] = []
|
||||
for s in sections:
|
||||
piece = Section(s.page, [], s.heading)
|
||||
for line in s.lines:
|
||||
if piece.lines and size([Section(s.page, piece.lines + [line], piece.heading)]) > room:
|
||||
chunks.append(piece)
|
||||
piece = Section(s.page, [], s.heading + " (continued)")
|
||||
piece.lines.append(line)
|
||||
if size([piece]) > room:
|
||||
raise RunError(f"one sentence of {s.page} does not fit in a part of {part_chars} characters")
|
||||
chunks.append(piece)
|
||||
groups: list[list[Section]] = []
|
||||
for c in chunks:
|
||||
if groups and size(groups[-1] + [c]) <= room:
|
||||
groups[-1].append(c)
|
||||
else:
|
||||
groups.append([c])
|
||||
out: list[tuple[str, dict[str, Any]]] = []
|
||||
for i, group in enumerate(groups, 1):
|
||||
text = f"<!-- part {i} of {len(groups)} -->\n" + part_text(label, fixed, group)
|
||||
out.append((text, {"part": i, "file": f"part-{i}.md", "pages": len({s.page for s in group}),
|
||||
"sentences": sum(len(s.lines) for s in group), "catalogue_chars": len(fixed),
|
||||
"chars": len(text)}))
|
||||
return out
|
||||
|
||||
|
||||
def sentence_line(s: Sentence) -> str:
|
||||
return f"- {s.key} | {' '.join(s.text.split())}\n"
|
||||
|
||||
|
||||
def render_parts(bundle: Path, wanted: dict[str, list[str]], catalogue: list[str], label: str,
|
||||
part_chars: int = PART_CHARS) -> list[tuple[str, dict[str, Any]]]:
|
||||
return render_pages(read_pages(bundle), wanted, catalogue, label, part_chars)
|
||||
|
||||
|
||||
def render_pages(pages: dict[str, Page], wanted: dict[str, list[str]], catalogue: list[str], label: str,
|
||||
part_chars: int = PART_CHARS) -> list[tuple[str, dict[str, Any]]]:
|
||||
sections = []
|
||||
for rel, keys in wanted.items():
|
||||
if rel not in pages:
|
||||
raise RunError(f"{rel} is not a concept page in the bundle")
|
||||
by_key = {s.key: s for s in body_sentences(pages[rel].body)}
|
||||
missing = [k for k in keys if k not in by_key]
|
||||
if missing:
|
||||
raise RunError(f"{rel}: sentence keys not on the page: {', '.join(missing[:3])}")
|
||||
sections.append(Section(rel, [sentence_line(by_key[k]) for k in keys], rel))
|
||||
return render_sections(label, catalogue, sections, pages, part_chars)
|
||||
|
||||
|
||||
def write_parts(out: Path, parts: list[tuple[str, dict[str, Any]]], extra: dict[str, Any] | None = None) -> None:
|
||||
paths = [out / str(stats["file"]) for _, stats in parts] + [out / "parts.json"]
|
||||
if any(p.exists() for p in paths):
|
||||
raise RunError(f"task parts exist in {out}; refusing to overwrite")
|
||||
for (text, _), p in zip(parts, paths):
|
||||
write_new(p, text)
|
||||
write_new(out / "parts.json", dump({"schema": PARTS_SCHEMA, **(extra or {}), "parts": [s for _, s in parts]}))
|
||||
|
||||
|
||||
def read_ids(path: Path) -> list[str]:
|
||||
ids: list[str] = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
cid = line.strip().removesuffix(".md")
|
||||
if cid and cid not in ids:
|
||||
ids.append(cid)
|
||||
return ids
|
||||
|
||||
|
||||
# --- the guard -------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def kilde_path(text: str) -> str | None:
|
||||
"""The first K1 path (kilde/...) in text, with or without a leading /."""
|
||||
m = KILDE_PATH.search(text)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def homeless(audit_doc: Any) -> tuple[list[str], list[str]]:
|
||||
"""(pages with no-main-page-link, K1 paths with main-page-missing) from the auditor's JSON, each once, sorted.
|
||||
|
||||
The auditor is the only judge (K2 point 17): nothing here counts links. A main-page-missing finding names its
|
||||
source in `file` or, failing that, in `message`; one that names no kilde/ path cannot run."""
|
||||
findings = audit_doc.get("findings") if isinstance(audit_doc, dict) else None
|
||||
if not isinstance(findings, list) or not all(isinstance(f, dict) for f in findings):
|
||||
raise RunError("the auditor's output has no list of findings")
|
||||
pages: set[str] = set()
|
||||
missing: set[str] = set()
|
||||
for f in findings:
|
||||
if f.get("check") == "no-main-page-link":
|
||||
pages.add(page_rel(str(f.get("file") or "")))
|
||||
elif f.get("check") == "main-page-missing":
|
||||
path = kilde_path(str(f.get("file") or "")) or kilde_path(str(f.get("message") or ""))
|
||||
if path is None:
|
||||
raise RunError(f"a main-page-missing finding names no kilde/ source: {f!r}")
|
||||
missing.add(path)
|
||||
return sorted(pages), sorted(missing)
|
||||
|
||||
|
||||
def guard_candidates(bundle: Path, sources: list[str]) -> dict[str, list[str]]:
|
||||
"""Per K1 path, the ids of the pages that cite it: the only pages that can be its main page (K2 point 1)."""
|
||||
pages = read_pages(bundle)
|
||||
return {src: sorted(concept_id(rel) for rel, p in pages.items() if "/" + src in p.resources.values())
|
||||
for src in sources}
|
||||
|
||||
|
||||
def guard_line(cid: str, pages: dict[str, Page], described: bool) -> str:
|
||||
rel = page_rel(cid)
|
||||
if rel not in pages:
|
||||
raise RunError(f"{rel} is not a concept page in the bundle")
|
||||
p = pages[rel]
|
||||
return f"- {cid} | {' '.join(p.title.split())}" + (f" | {p.description}" if described and p.description else "") + "\n"
|
||||
|
||||
|
||||
def guard_parts(bundle: Path, homeless_pages: list[str], sources: list[str], catalogue: list[str], label: str,
|
||||
part_chars: int = PART_CHARS) -> list[str]:
|
||||
"""The guard's task: the contract and the catalogue of main pages in every part, then the pages without a home
|
||||
and the sources without a main page (each with the pages that cite it), in parts of at most part_chars."""
|
||||
pages = read_pages(bundle)
|
||||
candidates = guard_candidates(bundle, sources)
|
||||
head = (f"# Home task: {label}\n\n" + GUARD_CONTRACT + "\n## Catalogue (main pages)\n\n"
|
||||
+ "".join(guard_line(cid, pages, False) for cid in catalogue) + "\n")
|
||||
items: list[tuple[str, str]] = [("pages", guard_line(concept_id(page_rel(rel)), pages, True))
|
||||
for rel in homeless_pages]
|
||||
for src in sources:
|
||||
cands = "".join(guard_line(cid, pages, True) for cid in candidates[src]) or "(no page cites it)\n"
|
||||
items.append(("sources", f"### {src}\n\n{cands}\n"))
|
||||
titles = {"pages": "## Pages without a home\n\n", "sources": "## Sources without a main page\n\n"}
|
||||
|
||||
def text(group: list[tuple[str, str]]) -> str:
|
||||
out, kind = head, ""
|
||||
for k, piece in group:
|
||||
if k != kind:
|
||||
out, kind = out + ("\n" if kind else "") + titles[k], k
|
||||
out += piece
|
||||
return out
|
||||
|
||||
room = part_chars - PART_HEADER_ROOM
|
||||
if len(text([])) > room:
|
||||
raise RunError(f"the guard's contract and catalogue leave no room for a page within {part_chars} characters")
|
||||
groups: list[list[tuple[str, str]]] = []
|
||||
for item in items:
|
||||
if groups and len(text(groups[-1] + [item])) <= room:
|
||||
groups[-1].append(item)
|
||||
elif len(text([item])) <= room:
|
||||
groups.append([item])
|
||||
else:
|
||||
raise RunError(f"one item of the guard's task does not fit in a part of {part_chars} characters")
|
||||
return [f"<!-- part {i} of {len(groups)} -->\n" + text(g) for i, g in enumerate(groups, 1)]
|
||||
|
||||
|
||||
@dataclass
|
||||
class GuardAnswer:
|
||||
homes: list[tuple[str, str]] # (page id, home id)
|
||||
main_pages: list[tuple[str, str]] # (K1 path, page id)
|
||||
none: list[str]
|
||||
unanswered: list[str]
|
||||
rejected: Counter[str]
|
||||
|
||||
|
||||
def read_guard(answer: Any, asked_pages: list[str], asked_sources: list[str], catalogue: list[str],
|
||||
candidates: dict[str, list[str]]) -> GuardAnswer:
|
||||
"""The guard's answer as code accepts it: per source a main page among the pages that cite it; per page a home
|
||||
from the catalogue or from the main pages this answer names, or none; the first valid answer for each counts."""
|
||||
doc = answer if isinstance(answer, dict) else {}
|
||||
asked = [concept_id(page_rel(p)) for p in asked_pages]
|
||||
rejected: Counter[str] = Counter()
|
||||
|
||||
def entries(key: str) -> list[Any]:
|
||||
value = doc.get(key)
|
||||
return value if isinstance(value, list) else []
|
||||
|
||||
main_pages: list[tuple[str, str]] = []
|
||||
for m in entries("main_pages"):
|
||||
if not (isinstance(m, dict) and isinstance(m.get("source"), str) and isinstance(m.get("page"), str)):
|
||||
rejected["malformed"] += 1
|
||||
continue
|
||||
src, pid = kilde_path(m["source"]), concept_id(page_rel(m["page"]))
|
||||
if src not in asked_sources:
|
||||
rejected["source-not-asked"] += 1
|
||||
elif any(s == src for s, _ in main_pages):
|
||||
rejected["duplicate"] += 1
|
||||
elif pid not in candidates.get(src, []):
|
||||
rejected["page-not-a-candidate"] += 1
|
||||
else:
|
||||
main_pages.append((src, pid))
|
||||
named = {pid for _, pid in main_pages}
|
||||
targets = set(catalogue) | named
|
||||
homes: list[tuple[str, str]] = []
|
||||
answered: set[str] = set()
|
||||
for h in entries("homes"):
|
||||
if not (isinstance(h, dict) and isinstance(h.get("page"), str) and isinstance(h.get("home"), str)):
|
||||
rejected["malformed"] += 1
|
||||
continue
|
||||
pid, hid = concept_id(page_rel(h["page"])), concept_id(page_rel(h["home"]))
|
||||
if pid not in asked:
|
||||
rejected["page-not-asked"] += 1
|
||||
elif pid in answered:
|
||||
rejected["duplicate"] += 1
|
||||
elif hid == pid:
|
||||
rejected["self-home"] += 1
|
||||
elif hid not in targets:
|
||||
rejected["home-not-in-catalogue"] += 1
|
||||
else:
|
||||
homes.append((pid, hid))
|
||||
answered.add(pid)
|
||||
none: list[str] = []
|
||||
for x in entries("none"):
|
||||
if not isinstance(x, str):
|
||||
rejected["malformed"] += 1
|
||||
continue
|
||||
pid = concept_id(page_rel(x))
|
||||
if pid not in asked:
|
||||
rejected["page-not-asked"] += 1
|
||||
elif pid in answered:
|
||||
rejected["duplicate"] += 1
|
||||
else:
|
||||
none.append(pid)
|
||||
answered.add(pid)
|
||||
unanswered = [pid for pid in asked if pid not in answered and pid not in named]
|
||||
return GuardAnswer(homes, main_pages, none, unanswered, rejected)
|
||||
|
||||
|
||||
# --- accept ----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_anchor(s: Sentence, anchor: str) -> tuple[int | None, str | None]:
|
||||
"""(offset of the first occurrence outside code, None) or (None, reason)."""
|
||||
if not anchor or any(c in anchor for c in "[]\n"):
|
||||
return None, "anchor-not-in-sentence"
|
||||
in_code = False
|
||||
start = s.text.find(anchor)
|
||||
while start >= 0:
|
||||
end = start + len(anchor)
|
||||
whole = (start == 0 or not s.text[start - 1].isalnum()) and (end == len(s.text) or not s.text[end].isalnum())
|
||||
if whole:
|
||||
if s.masked[start:end] == anchor:
|
||||
return start, None
|
||||
in_code = True
|
||||
start = s.text.find(anchor, start + 1)
|
||||
return None, "anchor-in-code" if in_code else "anchor-not-in-sentence"
|
||||
|
||||
|
||||
def accept(proposals: dict[str, Any], bundle: Path, links: Path, origin: str) -> dict[str, Any]:
|
||||
pages = read_pages(bundle)
|
||||
sentences = {rel: {s.key: s for s in body_sentences(p.body)} for rel, p in pages.items()}
|
||||
doc: dict[str, Any] = load_json(links) if links.is_file() else {
|
||||
"schema": SCHEMA, "links": [], "none": [], "none_count": 0, "accepted": 0, "rejected": {}}
|
||||
records: list[dict[str, Any]] = doc["links"]
|
||||
rejected: Counter[str] = Counter()
|
||||
accepted = 0
|
||||
raw = proposals.get("links") if isinstance(proposals, dict) else None
|
||||
for p in raw if isinstance(raw, list) else []:
|
||||
reason, record = judge(p, pages, sentences, records)
|
||||
if reason:
|
||||
rejected[reason] += 1
|
||||
else:
|
||||
assert record is not None
|
||||
records.append(record)
|
||||
accepted += 1
|
||||
none = proposals.get("none") if isinstance(proposals, dict) else None
|
||||
nones = [str(x) for x in none] if isinstance(none, list) else []
|
||||
doc["none"] += [{"id": x, "from": origin} for x in nones]
|
||||
doc["none_count"] = len(doc["none"])
|
||||
doc["accepted"] = len(records)
|
||||
total = Counter(doc["rejected"])
|
||||
total.update(rejected)
|
||||
doc["rejected"] = {r: total[r] for r in REASONS if total[r]}
|
||||
links.parent.mkdir(parents=True, exist_ok=True)
|
||||
links.write_text(dump(doc), encoding="utf-8")
|
||||
return {"accepted": accepted, "rejected": dict(rejected), "none": len(nones)}
|
||||
|
||||
|
||||
def judge(p: Any, pages: dict[str, Page], sentences: dict[str, dict[str, Sentence]],
|
||||
records: list[dict[str, Any]]) -> tuple[str | None, dict[str, Any] | None]:
|
||||
if not (isinstance(p, dict) and all(isinstance(p.get(k), str) for k in ("page", "sentence", "anchor", "target"))):
|
||||
return "malformed", None
|
||||
rel, target, anchor, key = page_rel(p["page"]), page_rel(p["target"]), p["anchor"], p["sentence"]
|
||||
s = sentences.get(rel, {}).get(key)
|
||||
if s is None:
|
||||
return "sentence-missing", None
|
||||
if target not in pages:
|
||||
return "target-missing", None
|
||||
if target == rel:
|
||||
return "self-link", None
|
||||
if not s.footnoted:
|
||||
return "no-footnote", None
|
||||
at, reason = find_anchor(s, anchor)
|
||||
if at is None:
|
||||
return reason, None
|
||||
here = [r for r in records if r["page"] == rel and r["sentence"] == key]
|
||||
if any(r["target"] == target for r in here):
|
||||
return "duplicate", None
|
||||
if len(here) >= MAX_LINKS:
|
||||
return "over-limit", None
|
||||
if any(at < int(r["at"]) + len(str(r["anchor"])) and int(r["at"]) < at + len(anchor) for r in here):
|
||||
return "overlap", None
|
||||
return None, {"page": rel, "sentence": key, "anchor": anchor, "at": at, "target": target}
|
||||
|
||||
|
||||
# --- command line ----------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
s = sub.add_parser("strip")
|
||||
s.add_argument("bundle")
|
||||
s.add_argument("--out", required=True)
|
||||
se = sub.add_parser("sentences")
|
||||
for name in ("--accepted", "--bundle", "--out"):
|
||||
se.add_argument(name, required=True)
|
||||
se.add_argument("--pack", type=int, required=True)
|
||||
t = sub.add_parser("task")
|
||||
t.add_argument("bundle")
|
||||
for name in ("--sentences", "--catalogue", "--out"):
|
||||
t.add_argument(name, required=True)
|
||||
a = sub.add_parser("accept")
|
||||
a.add_argument("proposals")
|
||||
a.add_argument("--bundle", required=True)
|
||||
a.add_argument("--links", required=True)
|
||||
pl = sub.add_parser("place")
|
||||
pl.add_argument("bundle")
|
||||
pl.add_argument("--links", required=True)
|
||||
pl.add_argument("--out", required=True)
|
||||
pl.add_argument("--replace", action="store_true")
|
||||
b = sub.add_parser("pass-before")
|
||||
for name in ("--accepted", "--bundle", "--out"):
|
||||
b.add_argument(name, required=True)
|
||||
b.add_argument("--catalogue")
|
||||
b.add_argument("--pack", type=int, required=True)
|
||||
b.add_argument("--to", type=int)
|
||||
f = sub.add_parser("pass-after")
|
||||
for name in ("--pass-dir", "--bundle", "--links", "--linked", "--sources", "--transcripts", "--okter", "--work",
|
||||
"--kost", "--lines", "--unit"):
|
||||
f.add_argument(name, required=True)
|
||||
for name in ("--cap", "--done", "--total"):
|
||||
f.add_argument(name, type=int, required=True)
|
||||
f.add_argument("--blocks")
|
||||
f.add_argument("--name")
|
||||
return p
|
||||
|
||||
|
||||
def sentences_doc(accepted: Path, pack: int, bundle: Path) -> dict[str, Any]:
|
||||
wanted = pack_sentences(load_json(accepted), pack, read_pages(bundle))
|
||||
return {"schema": SENTENCES_SCHEMA, "pack": pack, "pages": wanted}
|
||||
|
||||
|
||||
def pass_before(args: argparse.Namespace) -> int:
|
||||
accepted, pages = load_json(Path(args.accepted)), read_pages(Path(args.bundle))
|
||||
given = read_ids(Path(args.catalogue)) if args.catalogue else None
|
||||
last = args.pack if args.to is None else args.to
|
||||
if last < args.pack:
|
||||
raise RunError(f"--to {last} is before --pack {args.pack}")
|
||||
jobs = []
|
||||
for pack in range(args.pack, last + 1): # render every pack before writing any, so a failure leaves nothing behind
|
||||
out = Path(args.out) if args.to is None else Path(args.out) / f"p{pack}"
|
||||
wanted = pack_sentences(accepted, pack, pages)
|
||||
pool = given if given is not None else pack_pool(accepted, pack, pages)
|
||||
ids = filter_catalogue(accepted, pack, pool, pages)
|
||||
jobs.append((pack, out, wanted, pool, ids, render_pages(pages, wanted, ids, f"pack {pack}")))
|
||||
empty = 0
|
||||
for pack, out, wanted, pool, ids, parts in jobs:
|
||||
write_new(out / "sentences.json", dump({"schema": SENTENCES_SCHEMA, "pack": pack, "pages": wanted}))
|
||||
write_parts(out, parts, {"pack": pack, "catalogue": {"concepts": len(ids), "pool": len(pool)}})
|
||||
n = sum(len(v) for v in wanted.values())
|
||||
empty += not n
|
||||
cat = parts[0][1]["catalogue_chars"] if parts else 0
|
||||
print(f"pass-before: pack {pack}, pages {len(wanted)}, sentences {n}, parts {len(parts)}, catalogue "
|
||||
f"{len(ids)} of {len(pool)} concepts, {cat} characters -> {out}")
|
||||
for _, stats in parts:
|
||||
print(f" {out / str(stats['file'])} -> {out / ('proposals-' + str(stats['part']) + '.json')}")
|
||||
return 1 if empty else 0
|
||||
|
||||
|
||||
def counts(rejected: dict[str, int]) -> str:
|
||||
return ", ".join(f"{r} {n}" for r, n in sorted(rejected.items())) or "none"
|
||||
|
||||
|
||||
def pack_dirs(pass_dir: Path) -> list[Path]:
|
||||
"""The p<n> directories of an interval, in pack order."""
|
||||
dirs = [d for d in pass_dir.iterdir() if d.is_dir() and re.fullmatch(r"p\d+", d.name)]
|
||||
return sorted(dirs, key=lambda d: int(d.name[1:]))
|
||||
|
||||
|
||||
def accept_dir(directory: Path, bundle: Path, links: Path) -> tuple[int, int, Counter[str]]:
|
||||
files = sorted(directory.glob("proposals-*.json"))
|
||||
if not files:
|
||||
raise RunError(f"no proposals-*.json in {directory}")
|
||||
totals: Counter[str] = Counter()
|
||||
accepted = nones = 0
|
||||
for path in files:
|
||||
r = accept(load_json(path), bundle, links, str(path))
|
||||
accepted, nones = accepted + int(r["accepted"]), nones + int(r["none"])
|
||||
totals.update(r["rejected"])
|
||||
return accepted, nones, totals
|
||||
|
||||
|
||||
def stop_blocks(kost: dict[str, Any], packs_log: dict[str, Any], lines: list[str], last: int,
|
||||
done: int) -> tuple[int, list[int], int, int | None]:
|
||||
"""The blocks done after a pass whose last pack is `last`: the packs up to `last` and every pack with a compiler
|
||||
run in the harvest, so the measured cost and the blocks cover the same packs (order 22 session B2: the next pack's
|
||||
compiler counted over the pass's blocks alone). `done` must be the blocks of the accepted packs up to `last`.
|
||||
Second: the packs with a compiler run in this ledger, over whose blocks the stop line takes the next pack's share
|
||||
(order 27 A3: packs built in an earlier ledger cost nothing here). Third: the blocks of the compiled packs not yet
|
||||
accepted, whose compiler may run once more before the next check. Fourth: their characters (`block_chars`),
|
||||
None when one of them has none (a log written before blocks had characters; motor B17)."""
|
||||
blocks = {int(p["pack"]): int(p["blocks"]) for p in packs_log["packs"]}
|
||||
covered = sum(int(p["blocks"]) for p in packs_log["packs"] if "accepted" in p and int(p["pack"]) <= last)
|
||||
if done != covered:
|
||||
raise RunError(f"--done {done} is not the blocks of the accepted packs up to pack {last} ({covered})")
|
||||
compiled: set[int] = set()
|
||||
for r in kost["runs"]:
|
||||
if r["line"] not in lines:
|
||||
continue
|
||||
if r.get("pack") is None:
|
||||
raise RunError(f"the run {r['description']!r} is tied to no pack: STOP")
|
||||
if int(r["pack"]) not in blocks:
|
||||
raise RunError(f"the run {r['description']!r} is for pack {r['pack']}, which the pack log lacks: STOP")
|
||||
if r["line"] == "build":
|
||||
compiled.add(int(r["pack"]))
|
||||
built = {n for n in blocks if n <= last} | compiled
|
||||
waiting = [p for p in packs_log["packs"] if int(p["pack"]) in built and "accepted" not in p]
|
||||
pending = sum(int(p["blocks"]) for p in waiting)
|
||||
chars = (sum(int(p["block_chars"]) for p in waiting)
|
||||
if all(isinstance(p.get("block_chars"), int) for p in waiting) else None)
|
||||
return sum(blocks[n] for n in built), sorted(compiled), pending, chars
|
||||
|
||||
|
||||
def pass_after(args: argparse.Namespace) -> int:
|
||||
pass_dir, bundle, links = Path(args.pass_dir), Path(args.bundle), Path(args.links)
|
||||
dirs = pack_dirs(pass_dir)
|
||||
for d in dirs: # every pack has its proposals before any is accepted, so a missing one changes nothing
|
||||
if not any(d.glob("proposals-*.json")):
|
||||
raise RunError(f"no proposals-*.json in {d}")
|
||||
totals: Counter[str] = Counter()
|
||||
accepted = nones = 0
|
||||
for d in dirs or [pass_dir]:
|
||||
a, n, t = accept_dir(d, bundle, links)
|
||||
if dirs:
|
||||
print(f"accept {d.name}: accepted {a}, rejected {sum(t.values())} ({counts(dict(t))}), none {n}")
|
||||
accepted, nones = accepted + a, nones + n
|
||||
totals.update(t)
|
||||
print(f"accept: accepted {accepted}, rejected {sum(totals.values())} ({counts(dict(totals))}), none {nones}")
|
||||
placed, stale = place(bundle, links, Path(args.linked), replace=True)
|
||||
print(f"place: {placed} links placed, {stale} stale")
|
||||
if stale:
|
||||
return 1
|
||||
report = audit.link_rule_report(audit.load_bundle(Path(args.linked), Path(args.sources)))
|
||||
print(f"link rule: {report['pass']} of {report['of']}, misplaced {report['misplaced']}, broken {report['broken']}, "
|
||||
f"max out-degree {report['max_out_degree']}")
|
||||
harvest = ["harvest", "--transcripts", args.transcripts, "--okter", args.okter, "--work", args.work,
|
||||
"--out", args.kost]
|
||||
code = cost.main(harvest)
|
||||
print(f"cost harvest: exit {code}")
|
||||
if code:
|
||||
return code
|
||||
done, following, chars = args.done, None, None
|
||||
if args.unit == "blocks":
|
||||
if not args.blocks:
|
||||
raise RunError("--unit blocks needs --blocks (the pack log)")
|
||||
last = max(int(d.name[1:]) for d in dirs) if dirs else int(load_json(pass_dir / "sentences.json")["pack"])
|
||||
done, packs, following, chars = stop_blocks(load_json(Path(args.kost)), load_json(Path(args.blocks)),
|
||||
cost.capped_lines(args.lines), last, args.done)
|
||||
print(f"done: {done} of {args.total} blocks (packs {', '.join(map(str, packs))} with a compiler run)")
|
||||
stop = ["stop", "--lines", args.lines, "--cap", str(args.cap), "--unit", args.unit, "--done", str(done),
|
||||
"--total", str(args.total), "--kost", args.kost]
|
||||
stop += ["--blocks", args.blocks] if args.blocks else []
|
||||
stop += ["--name", args.name] if args.name else []
|
||||
stop += ["--next", str(following)] if following is not None else []
|
||||
stop += ["--next-chars", str(chars)] if following is not None and chars is not None else []
|
||||
code = cost.main(stop)
|
||||
print(f"cost stop: exit {code}")
|
||||
return code
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
if args.command == "strip":
|
||||
n = strip(Path(args.bundle), Path(args.out))
|
||||
print(f"strip: {n} links reduced to their anchor text -> {args.out}")
|
||||
return 0
|
||||
if args.command == "sentences":
|
||||
doc = sentences_doc(Path(args.accepted), args.pack, Path(args.bundle))
|
||||
write_new(Path(args.out), dump(doc))
|
||||
n = sum(len(v) for v in doc["pages"].values())
|
||||
print(f"sentences: pack {args.pack}, {len(doc['pages'])} pages, {n} sentences -> {args.out}")
|
||||
return 0 if n else 1
|
||||
if args.command == "pass-before":
|
||||
return pass_before(args)
|
||||
if args.command == "task":
|
||||
out = Path(args.out)
|
||||
doc = load_json(Path(args.sentences))
|
||||
label = Path(args.sentences).stem
|
||||
parts = render_parts(Path(args.bundle), doc["pages"], read_ids(Path(args.catalogue)), label)
|
||||
write_parts(out, parts)
|
||||
n = sum(len(v) for v in doc["pages"].values())
|
||||
cat = parts[0][1]["catalogue_chars"] if parts else 0
|
||||
print(f"{args.command}: pages {len(doc['pages'])}, sentences {n}, parts {len(parts)}, catalogue {cat} "
|
||||
f"characters -> {out}")
|
||||
for _, stats in parts:
|
||||
print(f" {out / str(stats['file'])} -> {out / ('proposals-' + str(stats['part']) + '.json')}")
|
||||
return 0 if n else 1
|
||||
if args.command == "accept":
|
||||
r = accept(load_json(Path(args.proposals)), Path(args.bundle), Path(args.links), args.proposals)
|
||||
print(f"accept: accepted {r['accepted']}, rejected {sum(r['rejected'].values())} "
|
||||
f"({counts(r['rejected'])}), none {r['none']}")
|
||||
return 0
|
||||
if args.command == "place":
|
||||
placed, stale = place(Path(args.bundle), Path(args.links), Path(args.out), args.replace)
|
||||
print(f"place: {placed} links placed, {stale} stale -> {args.out}")
|
||||
return 1 if stale else 0
|
||||
return pass_after(args)
|
||||
except (RunError, OSError, json.JSONDecodeError, KeyError, ValueError, TypeError) as exc:
|
||||
print(f"klink cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
1012
okfwiki/kmerge.py
Normal file
1012
okfwiki/kmerge.py
Normal file
File diff suppressed because it is too large
Load diff
1992
okfwiki/kpack.py
Normal file
1992
okfwiki/kpack.py
Normal file
File diff suppressed because it is too large
Load diff
1212
okfwiki/krepair.py
Normal file
1212
okfwiki/krepair.py
Normal file
File diff suppressed because it is too large
Load diff
546
okfwiki/reader.py
Normal file
546
okfwiki/reader.py
Normal file
|
|
@ -0,0 +1,546 @@
|
|||
"""The tool around the reader pass for PLAN.md criterion 2 ("slutning støttet"). It calls no model.
|
||||
|
||||
The reader judges up to ten claims per call and gives one verdict per claim (K3 in the wave-1 contracts). The model
|
||||
call happens in the user's own Claude Code session: one isolated subagent per call, described "V1-<run>-read-<call>",
|
||||
which reads one rendered task file and replies with one line per claim. The agent definition
|
||||
(agents/okf-wiki-reader.md) is thin: it only tells the subagent to read its task file and follow it. The instructions
|
||||
travel in the task (INSTRUCTIONS below), so a change between runs reaches the reader whatever the session loaded, and
|
||||
the transcript shows word for word what the reader was told.
|
||||
|
||||
render writes one task per call of a call layout (okf-wiki/reader-calls/1, bound to the set by its sha256):
|
||||
the instructions, then the claims numbered 1. to n. in layout order, each with its excerpts (file, line,
|
||||
text). Nothing else reaches the reader: no id, no call name, no label, no other call. It refuses to write
|
||||
when the agent's system text or the instructions share a run of eight words with any claim or excerpt of
|
||||
the set, so the reader cannot carry text from the pairs it is judged on, and it never changes a task
|
||||
already written.
|
||||
collect reads every reader's reply from the subagent transcripts (<transcripts>/<session>/subagents/, the
|
||||
or <session>/subagents/workflows/<id>/ for a workflow; the agent type okf-wiki-reader, bare or
|
||||
plugin-namespaced as <plugin>:okf-wiki-reader, and the
|
||||
description "V1-<run>-read-<call>"), the message it handed back or else its last text, and turns it
|
||||
into the verdicts file that gates/calibration.py scores ({"reader": "<name>", "verdicts": [{"id",
|
||||
"verdict"}]}). One rule, fail-closed (K3 point 3): every stripped, non-empty line must be
|
||||
"<1-10>: supported" or "<1-10>: unsupported"; one line that does not fit, or a number outside 1..n,
|
||||
gives no verdict for any claim of the call; a number given twice or not at all gives none for that
|
||||
claim. A call answered by two transcripts gets no verdict.
|
||||
It also names the models the readers ran on (PLAN § Rammer 23): the sorted, distinct `message.model`
|
||||
of the assistant records in the transcripts it turns into verdicts (records without one, and
|
||||
"<synthetic>", are left out), written as "models" beside "reader" and "verdicts" and printed as
|
||||
"models: ...". One of two choices is required: --calibration says this run is a calibration gate run
|
||||
and is not checked; --gate-passed FILE names the record of green gate runs (gates/calibration.py gate
|
||||
--passed; --gate-passed alone reads the tracked gates/calibration.passed.json, which holds only models
|
||||
and sha256, so a fresh clone without the set can still read), and when the transcripts name no model, or a model the record does not hold with a run on the
|
||||
set and key sha256 of every calibration manifest (--manifest, once per manifest; default the tracked
|
||||
gates/calibration.manifest.json, the Claude Code docs set, and gates/calibration-folder.manifest.json,
|
||||
the folder set of Norwegian and English PDF/Office documents), collect writes nothing and exits 1, so a
|
||||
reading on a model that never passed both gates gives no verdicts.
|
||||
|
||||
Exit 0 = written with every verdict; 1 = written with verdicts missing, refused, or could not run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from okfwiki.transcript import is_agent_type, read_transcript
|
||||
|
||||
SUPPORTED = "supported"
|
||||
UNSUPPORTED = "unsupported"
|
||||
CALLS_SCHEMA = "okf-wiki/reader-calls/1"
|
||||
CALL_SIZE_MAX = 10
|
||||
RUN_WORDS = 8
|
||||
WORD = re.compile(r"[a-z0-9]+(?:['’.-][a-z0-9]+)*")
|
||||
LINE = re.compile(r"([1-9]|10)\s*:\s*(supported|unsupported)")
|
||||
CALL_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*")
|
||||
HANDBACK = "SubagentHandback"
|
||||
AGENT_TYPE = "okf-wiki-reader"
|
||||
SYNTHETIC_MODEL = "<synthetic>"
|
||||
PASSED_SCHEMA = "okf-wiki/calibration-passed/1"
|
||||
CALIBRATION_MANIFEST = Path(__file__).resolve().parents[1] / "gates" / "calibration.manifest.json"
|
||||
CALIBRATION_FOLDER_MANIFEST = Path(__file__).resolve().parents[1] / "gates" / "calibration-folder.manifest.json"
|
||||
CALIBRATION_PASSED = Path(__file__).resolve().parents[1] / "gates" / "calibration.passed.json" # models and sha256 only
|
||||
|
||||
INSTRUCTIONS = """\
|
||||
# How to judge the claims
|
||||
|
||||
You are an independent reader. This task gives you a numbered list of claims from a knowledge collection. Under each
|
||||
claim stand the source excerpts it cites. For every claim, decide whether its own excerpts support it, and give one
|
||||
verdict per claim.
|
||||
|
||||
Judge each claim on its own. Only the excerpts printed under a claim count as evidence for that claim. The other claims
|
||||
in this task and their excerpts are not evidence for it, and the verdict on one claim must never lean on the verdict on
|
||||
another. Any number of the claims may be unsupported: none, one, several or all of them. Do not expect a pattern, and
|
||||
do not let a run of supported claims lower your guard on the next one.
|
||||
|
||||
You have not seen the pages the claims come from, and you must not look for them. What you know about the subject from
|
||||
anywhere else does not count, even when you are sure it is true: a claim that is true in the world but not stated by
|
||||
its excerpts is unsupported.
|
||||
|
||||
## What "supported" means
|
||||
|
||||
A claim is **supported** when everything it says is stated by its excerpts, or follows from them in one step that no
|
||||
careful reader of the excerpts could deny. Otherwise it is **unsupported**. There is no middle verdict: a claim that is
|
||||
mostly right is unsupported.
|
||||
|
||||
Allowed, and still supported:
|
||||
- Different words for the same thing: synonyms, reordering, active for passive, a plain-language summary.
|
||||
- Saying less than the excerpt: leaving out details, examples, conditions that only narrow what the excerpt says
|
||||
further, or items from a list, as long as what remains is still exactly true. A claim that is less specific than its
|
||||
source is fine; a claim that is more general is not.
|
||||
- One necessary step: a conclusion that the excerpt forces (if it says every X has Y and that Z is an X, then Z has Y;
|
||||
if it gives two numbers, their plain sum or difference). If the step needs any assumption of your own, it is not
|
||||
necessary.
|
||||
- Combining excerpts: when the claim joins facts from two excerpts, each part is backed by the excerpt that states it,
|
||||
and the join itself adds nothing new.
|
||||
|
||||
## How to read one claim
|
||||
|
||||
1. Break the claim into its parts: every subject, every thing said about it, every number and unit, every qualifier
|
||||
(all, always, only, never, some, may, by default, at most, at least, when, unless), every stated reason or order of
|
||||
events, and every negation.
|
||||
2. For each part, find the words in this claim's own excerpts that state it. If you cannot point to them, that part is
|
||||
not supported.
|
||||
3. Then run the six checks below on the claim as a whole. Each one describes a way a claim fails even though it sounds
|
||||
close to its source. One failed check makes the claim unsupported.
|
||||
|
||||
## The six checks
|
||||
|
||||
1. **Broader than the source.** The claim widens what the excerpt says: "some", "can", "may", "often", "by default" or
|
||||
"in this case" becomes "all", "always", "must", "every" or "in general"; a condition or exception in the excerpt is
|
||||
dropped so the claim covers cases the excerpt does not; one example is turned into a rule; one product, platform,
|
||||
version or setting is turned into all of them.
|
||||
2. **A number that differs.** Every number, count, limit, size, duration, date, version, percentage and unit in the
|
||||
claim must be the one in the excerpts, or the result of one plain calculation on them. A different figure, a rounded
|
||||
figure presented as exact, a swapped unit, or "at least" where the excerpt says "at most" fails. When the claim
|
||||
gives a figure the excerpts do not print (a difference, sum, total, share, ratio, count of listed items, or a span
|
||||
between two dates), work it out yourself from the excerpts' own figures before you judge; if your result is not
|
||||
the claim's figure, the claim fails, however close it is and however natural the sentence reads.
|
||||
3. **Cause, purpose or order that the source does not give.** The claim says A causes, enables, requires, prevents or
|
||||
happens before B, or that A is done in order to get B. The excerpts must state that same link in the same direction.
|
||||
Two facts that the excerpt only lists side by side are not a cause. A reversed direction (B causes A, B is needed
|
||||
for A) fails.
|
||||
4. **Something not in the excerpts.** Any subject, feature, option, value, reason, consequence, comparison or
|
||||
recommendation in the claim that the excerpts do not mention fails, however plausible it is. Look hardest at the
|
||||
last clause of the claim and at small added words ("also", "instead", "automatically", "only").
|
||||
5. **Negation or direction reversed.** The claim says the opposite of the excerpt on any point: can versus cannot, is
|
||||
versus is not, enabled versus disabled, allowed versus blocked, recommended versus discouraged, required versus
|
||||
optional, increases versus decreases, included versus excluded.
|
||||
6. **A property moved between different files.** When a claim's excerpts come from different files, they may be about
|
||||
different subjects. Each property in the claim must belong to the subject that its own excerpt attaches it to. A
|
||||
claim that gives one file's subject a property that only the other file states for its own subject fails, even
|
||||
when every word appears somewhere in the excerpts.
|
||||
|
||||
## Deciding
|
||||
|
||||
- If every part is backed and all six checks pass, the verdict is supported. Do not reject a claim for being short,
|
||||
for leaving things out, for style, or because you would have written it differently.
|
||||
- If any part is not backed, or any check fails, the verdict is unsupported.
|
||||
- When you are unsure whether a step is necessary or only likely, it is not necessary: unsupported.
|
||||
- Give every claim the same full reading, the last one as much as the first.
|
||||
|
||||
## Your reply
|
||||
|
||||
Work through every claim before you reply. Your reply is the list of verdicts and nothing else: one line per claim,
|
||||
`<number>: supported` or `<number>: unsupported`, where `<number>` is the claim's number in this task (1, 2, 3 and so
|
||||
on, without a leading zero) and the verdict is written in lower case. Give every number exactly once. Write nothing
|
||||
else in the reply: no heading, no explanation, no reasoning, no code block, no markup, and no text before or after the
|
||||
lines. Any other text in the reply makes the whole reply count as unanswered.
|
||||
|
||||
For a task with three claims, the reply has this form:
|
||||
|
||||
1: supported
|
||||
2: unsupported
|
||||
3: supported
|
||||
"""
|
||||
|
||||
|
||||
class RunError(Exception):
|
||||
"""The command cannot run; reported as FAIL with exit 1."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Excerpt:
|
||||
file: str
|
||||
line: int
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Pair:
|
||||
id: str
|
||||
claim: str
|
||||
excerpts: tuple[Excerpt, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Call:
|
||||
name: str
|
||||
ids: tuple[str, ...]
|
||||
|
||||
|
||||
def instructions(agent_text: str) -> str:
|
||||
"""The body of an agent definition, without its YAML frontmatter."""
|
||||
if agent_text.startswith("---\n"):
|
||||
end = agent_text.find("\n---\n", 4)
|
||||
if end < 0:
|
||||
raise RunError("the agent definition opens a frontmatter block that never closes")
|
||||
agent_text = agent_text[end + len("\n---\n") :]
|
||||
return agent_text.strip()
|
||||
|
||||
|
||||
def _read_json(path: Path, what: str) -> Any:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as exc:
|
||||
raise RunError(f"cannot read the {what} {path}: {exc}") from exc
|
||||
|
||||
|
||||
def load_pairs(path: Path) -> list[Pair]:
|
||||
"""Claims with their excerpts from a set file ({"pairs": [{id, claim, excerpts: [{file, line, text}]}]})."""
|
||||
data = _read_json(path, "set")
|
||||
raw = data.get("pairs") if isinstance(data, dict) else None
|
||||
if not isinstance(raw, list):
|
||||
raise RunError(f"{path} has no list of pairs")
|
||||
pairs: list[Pair] = []
|
||||
seen: set[str] = set()
|
||||
for p in raw:
|
||||
pid, claim, excerpts = (p.get(k) if isinstance(p, dict) else None for k in ("id", "claim", "excerpts"))
|
||||
if not isinstance(pid, str) or not isinstance(claim, str) or not isinstance(excerpts, list) or not excerpts:
|
||||
raise RunError(f"pair {pid!r} needs a string id, a string claim and at least one excerpt")
|
||||
if pid in seen:
|
||||
raise RunError(f"pair id {pid} is given twice")
|
||||
seen.add(pid)
|
||||
parsed = []
|
||||
for e in excerpts:
|
||||
if not (isinstance(e, dict) and isinstance(e.get("file"), str) and isinstance(e.get("text"), str)
|
||||
and isinstance(e.get("line"), int)):
|
||||
raise RunError(f"pair {pid}: every excerpt needs a file, a line and a text")
|
||||
parsed.append(Excerpt(e["file"], e["line"], e["text"]))
|
||||
pairs.append(Pair(pid, claim, tuple(parsed)))
|
||||
return pairs
|
||||
|
||||
|
||||
def load_calls(path: Path, set_path: Path) -> list[Call]:
|
||||
"""The calls of a layout (K3 point 1), bound to the set file by its sha256. A layout may hold part of the set
|
||||
(a sample); every id it names must be in the set and stand in exactly one call of at most call_size_max claims."""
|
||||
data = _read_json(path, "call layout")
|
||||
if not isinstance(data, dict) or data.get("schema") != CALLS_SCHEMA:
|
||||
raise RunError(f"{path} is not a call layout with schema {CALLS_SCHEMA}")
|
||||
size_max = data.get("call_size_max")
|
||||
if not isinstance(size_max, int) or isinstance(size_max, bool) or not 1 <= size_max <= CALL_SIZE_MAX:
|
||||
raise RunError(f"call_size_max {size_max!r} is not a whole number from 1 to {CALL_SIZE_MAX}")
|
||||
digest = hashlib.sha256(set_path.read_bytes()).hexdigest()
|
||||
if data.get("set_sha256") != digest:
|
||||
raise RunError(f"the layout is bound to set {data.get('set_sha256')!r}, not to {set_path} ({digest})")
|
||||
raw = data.get("calls")
|
||||
if not isinstance(raw, list) or not raw:
|
||||
raise RunError(f"{path} has no calls")
|
||||
known = {p.id for p in load_pairs(set_path)}
|
||||
calls: list[Call] = []
|
||||
names: set[str] = set()
|
||||
placed: set[str] = set()
|
||||
for c in raw:
|
||||
name, ids = (c.get("call"), c.get("ids")) if isinstance(c, dict) else (None, None)
|
||||
if not isinstance(name, str) or not CALL_NAME.fullmatch(name):
|
||||
raise RunError(f"call name {name!r} is not letters, digits, '-' and '_'")
|
||||
if name in names:
|
||||
raise RunError(f"call name {name} is given twice")
|
||||
names.add(name)
|
||||
if not isinstance(ids, list) or not ids or not all(isinstance(i, str) for i in ids):
|
||||
raise RunError(f"call {name} has no list of claim ids")
|
||||
if len(ids) > size_max:
|
||||
raise RunError(f"call {name} has {len(ids)} claims, more than {size_max}")
|
||||
for i in ids:
|
||||
if i not in known:
|
||||
raise RunError(f"call {name} names claim {i}, which is not in the set")
|
||||
if i in placed:
|
||||
raise RunError(f"claim {i} stands in more than one place")
|
||||
placed.add(i)
|
||||
calls.append(Call(name, tuple(ids)))
|
||||
return calls
|
||||
|
||||
|
||||
def render_call(call: Call, pairs: dict[str, Pair]) -> str:
|
||||
"""One task: the instructions, then the claims numbered 1. to n., each with its excerpts."""
|
||||
n = len(call.ids)
|
||||
parts = [INSTRUCTIONS.strip(), "", "---", "", "## The claims", "",
|
||||
f"This task has {n} claim{'s' if n != 1 else ''}, numbered 1 to {n}."]
|
||||
for number, cid in enumerate(call.ids, 1):
|
||||
p = pairs[cid]
|
||||
parts += ["", f"{number}. <claim>{p.claim}</claim>"]
|
||||
for e in p.excerpts:
|
||||
parts += [f'<excerpt file="{e.file}" line="{e.line}">', e.text, "</excerpt>"]
|
||||
return "\n".join(parts) + "\n"
|
||||
|
||||
|
||||
def _runs(text: str) -> set[tuple[str, ...]]:
|
||||
words = WORD.findall(text.lower())
|
||||
return {tuple(words[i : i + RUN_WORDS]) for i in range(len(words) - RUN_WORDS + 1)}
|
||||
|
||||
|
||||
def shared_runs(text: str, pairs: list[Pair]) -> list[str]:
|
||||
"""Pair ids whose claim or excerpts share a run of RUN_WORDS words with an instruction text."""
|
||||
mine = _runs(text)
|
||||
return [p.id for p in pairs if mine & _runs(" ".join([p.claim, *(e.text for e in p.excerpts)]))]
|
||||
|
||||
|
||||
def parse_reply(reply: str, n: int) -> dict[int, str]:
|
||||
"""K3 point 3: the verdict per claim number of one call of n claims; a claim with no verdict is left out."""
|
||||
lines = [line.strip() for line in reply.splitlines() if line.strip()]
|
||||
found: dict[int, list[str]] = {}
|
||||
for line in lines:
|
||||
m = LINE.fullmatch(line)
|
||||
if m is None:
|
||||
return {}
|
||||
number = int(m.group(1))
|
||||
if number > n:
|
||||
return {}
|
||||
found.setdefault(number, []).append(m.group(2))
|
||||
return {k: v[0] for k, v in sorted(found.items()) if len(v) == 1}
|
||||
|
||||
|
||||
def final_reply(path: Path) -> str:
|
||||
"""The reader's reply in its transcript: the message of its last SubagentHandback call (how Claude Code 2.1.295
|
||||
delivers a subagent's final report), or else the text of its last assistant message (its text blocks joined in
|
||||
order). Thinking is never part of the reply."""
|
||||
records = [r for r in read_transcript(path) if r.get("type") == "assistant" and isinstance(r.get("message"), dict)]
|
||||
handed: list[Any] = []
|
||||
for r in records:
|
||||
content = r["message"].get("content")
|
||||
for block in content if isinstance(content, list) else []:
|
||||
if isinstance(block, dict) and block.get("type") == "tool_use" and block.get("name") == HANDBACK:
|
||||
given = block.get("input")
|
||||
handed.append(given.get("message") if isinstance(given, dict) else None)
|
||||
if handed:
|
||||
return handed[-1] if isinstance(handed[-1], str) else ""
|
||||
if not records:
|
||||
return ""
|
||||
last = records[-1]["message"].get("id")
|
||||
texts: list[str] = []
|
||||
for r in records:
|
||||
if r["message"].get("id") != last:
|
||||
continue
|
||||
content = r["message"].get("content")
|
||||
for block in content if isinstance(content, list) else []:
|
||||
if isinstance(block, dict) and block.get("type") == "text" and isinstance(block.get("text"), str):
|
||||
texts.append(block["text"])
|
||||
return "\n".join(texts)
|
||||
|
||||
|
||||
def transcript_models(path: Path) -> set[str]:
|
||||
"""The models named by the assistant records of one transcript (message.model, as Claude Code writes it); records
|
||||
without one, and the "<synthetic>" records Claude Code writes itself, name none."""
|
||||
found: set[str] = set()
|
||||
for r in read_transcript(path):
|
||||
message = r.get("message") if r.get("type") == "assistant" else None
|
||||
model = message.get("model") if isinstance(message, dict) else None
|
||||
if isinstance(model, str) and model and model != SYNTHETIC_MODEL:
|
||||
found.add(model)
|
||||
return found
|
||||
|
||||
|
||||
PASSED_FIELDS = ("model", "set_sha256", "key_sha256")
|
||||
|
||||
|
||||
def read_passed(path: Path) -> list[dict[str, Any]]:
|
||||
"""The entries of a record of green calibration runs (okf-wiki/calibration-passed/1), the one parser both
|
||||
gates/calibration.py gate --passed (which adds to it) and collect --gate-passed use. A record that cannot be read,
|
||||
does not have that form, or has an entry without a model, set_sha256 or key_sha256 string, is a RunError."""
|
||||
data = _read_json(path, "record of green calibration runs")
|
||||
entries = data.get("passed") if isinstance(data, dict) and data.get("schema") == PASSED_SCHEMA else None
|
||||
if not isinstance(entries, list):
|
||||
raise RunError(f"{path} is not a record of green calibration runs with schema {PASSED_SCHEMA}")
|
||||
for e in entries:
|
||||
for name in PASSED_FIELDS:
|
||||
value = e.get(name) if isinstance(e, dict) else None
|
||||
if not isinstance(value, str) or not value:
|
||||
raise RunError(f"{path}: an entry of the record of green calibration runs names no {name}")
|
||||
return entries
|
||||
|
||||
|
||||
def manifest_shas(path: Path) -> tuple[str, str]:
|
||||
"""(set sha256, key sha256) of a calibration manifest (gates/calibration.manifest.json)."""
|
||||
data = _read_json(path, "calibration manifest")
|
||||
shas = [data.get(part, {}).get("sha256") if isinstance(data, dict) and isinstance(data.get(part), dict) else None
|
||||
for part in ("set", "key")]
|
||||
if not all(isinstance(s, str) and s for s in shas):
|
||||
raise RunError(f"{path} is not a calibration manifest with a set and a key sha256")
|
||||
return str(shas[0]), str(shas[1])
|
||||
|
||||
|
||||
def passed_models(path: Path, manifest: Path = CALIBRATION_MANIFEST) -> set[str]:
|
||||
"""The models with a green run on the set and key the calibration manifest names: an entry whose set_sha256 or
|
||||
key_sha256 differs (a toy set, an older set, a set rebuilt later) clears no model."""
|
||||
set_sha, key_sha = manifest_shas(manifest)
|
||||
return {e["model"] for e in read_passed(path) if e["set_sha256"] == set_sha and e["key_sha256"] == key_sha}
|
||||
|
||||
|
||||
def manifests_of(args: argparse.Namespace) -> list[Path]:
|
||||
"""The calibration manifests a green run must be on: those given with --manifest, else both tracked sets (the
|
||||
Claude Code docs set and the folder set of Norwegian and English PDF/Office documents, motor B16)."""
|
||||
given: list[str] | None = args.manifest
|
||||
return [Path(m) for m in given] if given else [CALIBRATION_MANIFEST, CALIBRATION_FOLDER_MANIFEST]
|
||||
|
||||
|
||||
def reader_transcripts(transcripts: Path, run: str) -> dict[str, list[Path]]:
|
||||
"""Transcripts per call name of the reader agents of one run, found by their description and agent type, whether
|
||||
started from the session (<session>/subagents/) or from a workflow (<session>/subagents/workflows/<id>/)."""
|
||||
prefix = re.compile(rf"^V1-{re.escape(run)}-read-(\S+)$")
|
||||
found: dict[str, list[Path]] = {}
|
||||
metas = [*transcripts.glob("*/subagents/agent-*.meta.json"),
|
||||
*transcripts.glob("*/subagents/workflows/*/agent-*.meta.json")]
|
||||
for meta_path in sorted(metas):
|
||||
try:
|
||||
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if not isinstance(meta, dict) or not is_agent_type(meta.get("agentType"), AGENT_TYPE):
|
||||
continue
|
||||
m = prefix.match(str(meta.get("description", "")))
|
||||
if m is None:
|
||||
continue
|
||||
path = meta_path.with_name(meta_path.name.removesuffix(".meta.json") + ".jsonl")
|
||||
found.setdefault(m.group(1), []).append(path)
|
||||
return found
|
||||
|
||||
|
||||
@dataclass
|
||||
class Collected:
|
||||
doc: dict[str, Any]
|
||||
claims: int
|
||||
answered: int = 0
|
||||
broke_rule: int = 0
|
||||
twice: int = 0
|
||||
unanswered: int = 0
|
||||
stray: int = 0
|
||||
|
||||
|
||||
def collect(name: str, calls: list[Call], found: dict[str, list[Path]]) -> Collected:
|
||||
verdicts: list[dict[str, str]] = []
|
||||
models: set[str] = set()
|
||||
out = Collected({"reader": name, "models": [], "verdicts": verdicts}, sum(len(c.ids) for c in calls))
|
||||
names = {c.name for c in calls}
|
||||
out.stray = sum(len(paths) for call, paths in found.items() if call not in names)
|
||||
for call in calls:
|
||||
paths = found.get(call.name, [])
|
||||
if not paths:
|
||||
out.unanswered += 1
|
||||
continue
|
||||
if len(paths) > 1:
|
||||
out.twice += 1
|
||||
continue
|
||||
out.answered += 1
|
||||
if paths[0].is_file():
|
||||
models |= transcript_models(paths[0])
|
||||
given = parse_reply(final_reply(paths[0]) if paths[0].is_file() else "", len(call.ids))
|
||||
if not given:
|
||||
out.broke_rule += 1
|
||||
for number, cid in enumerate(call.ids, 1):
|
||||
if number in given:
|
||||
verdicts.append({"id": cid, "verdict": given[number]})
|
||||
out.doc["models"] = sorted(models)
|
||||
return out
|
||||
|
||||
|
||||
def cmd_render(args: argparse.Namespace) -> int:
|
||||
set_path = Path(args.set)
|
||||
pairs = load_pairs(set_path)
|
||||
calls = load_calls(Path(args.calls), set_path)
|
||||
try:
|
||||
system = instructions(Path(args.agent).read_text(encoding="utf-8"))
|
||||
except OSError as exc:
|
||||
raise RunError(f"cannot read the agent definition {args.agent}: {exc}") from exc
|
||||
for what, text in (("the agent's system text", system), ("the instructions in the task", INSTRUCTIONS)):
|
||||
leaks = shared_runs(text, pairs)
|
||||
if leaks:
|
||||
raise RunError(f"{what} and the set share a run of {RUN_WORDS} words in {len(leaks)} pair(s); "
|
||||
"nothing written")
|
||||
by_id = {p.id: p for p in pairs}
|
||||
out = Path(args.out_dir)
|
||||
tasks = {out / f"{c.name}.md": render_call(c, by_id) for c in calls}
|
||||
changed = sorted(p.name for p, t in tasks.items() if p.exists() and p.read_text(encoding="utf-8") != t)
|
||||
if changed:
|
||||
raise RunError(f"{len(changed)} task(s) already written with other text, first {changed[0]}; nothing written")
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
for path, task in tasks.items():
|
||||
path.write_text(task, encoding="utf-8")
|
||||
print(f"render: {len(tasks)} tasks for {sum(len(c.ids) for c in calls)} claims written to {out}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_collect(args: argparse.Namespace) -> int:
|
||||
set_path = Path(args.set)
|
||||
calls = load_calls(Path(args.calls), set_path)
|
||||
transcripts = Path(args.transcripts)
|
||||
if not transcripts.is_dir():
|
||||
raise RunError(f"the transcripts directory {transcripts} does not exist")
|
||||
got = collect(args.reader, calls, reader_transcripts(transcripts, args.run))
|
||||
models: list[str] = got.doc["models"]
|
||||
if args.gate_passed is not None:
|
||||
for manifest in manifests_of(args):
|
||||
passed = passed_models(Path(args.gate_passed), manifest)
|
||||
lacking = [m for m in models if m not in passed] if models else ["(none)"]
|
||||
if lacking:
|
||||
print(f"collect cannot run: model(s) {', '.join(lacking)} have no green calibration run in "
|
||||
f"{args.gate_passed} on the set and key of {manifest}; run the calibration gate on them first",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
Path(args.out).write_text(json.dumps(got.doc, indent=1) + "\n", encoding="utf-8")
|
||||
written = len(got.doc["verdicts"])
|
||||
stray = f"; {got.stray} transcript{'s' if got.stray != 1 else ''} for a call not in the layout" if got.stray else ""
|
||||
print(f"collect {args.reader}: {written} of {got.claims} verdicts written to {args.out}")
|
||||
print(f"calls: {got.answered} of {len(calls)} answered; {got.broke_rule} broke the reply rule; "
|
||||
f"{got.twice} answered twice; {got.unanswered} not answered{stray}")
|
||||
print(f"missing: {got.claims - written}")
|
||||
print(f"models: {', '.join(models) if models else '(none)'}")
|
||||
return 0 if written == got.claims else 1
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
r = sub.add_parser("render", help="write one reader task per call")
|
||||
r.add_argument("--set", required=True)
|
||||
r.add_argument("--calls", required=True, help="the call layout (okf-wiki/reader-calls/1)")
|
||||
r.add_argument("--agent", required=True, help="the reader's agent definition (agents/okf-wiki-reader.md)")
|
||||
r.add_argument("--out-dir", required=True)
|
||||
c = sub.add_parser("collect", help="turn the readers' replies in the transcripts into a verdicts file")
|
||||
c.add_argument("--set", required=True)
|
||||
c.add_argument("--calls", required=True)
|
||||
c.add_argument("--transcripts", required=True,
|
||||
help="~/.claude/projects/<slug>/ of the session that ran the readers")
|
||||
c.add_argument("--run", required=True, help="the run name in the descriptions V1-<run>-read-<call>")
|
||||
c.add_argument("--reader", required=True)
|
||||
c.add_argument("--out", required=True)
|
||||
check = c.add_mutually_exclusive_group(required=True)
|
||||
check.add_argument("--calibration", action="store_true",
|
||||
help="this run is a calibration gate run; the readers' model is not checked")
|
||||
check.add_argument("--gate-passed", metavar="FILE", default=None, nargs="?", const=str(CALIBRATION_PASSED),
|
||||
help="the record of green calibration runs (gates/calibration.py gate --passed; without FILE "
|
||||
"the tracked gates/calibration.passed.json); collect writes nothing unless every model "
|
||||
"the readers ran on is in it")
|
||||
c.add_argument("--manifest", action="append", default=None,
|
||||
help="a calibration manifest whose set and key a green run must be on, once per manifest; every "
|
||||
"one must hold (default: the tracked gates/calibration.manifest.json and "
|
||||
"gates/calibration-folder.manifest.json)")
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
commands = {"render": cmd_render, "collect": cmd_collect}
|
||||
try:
|
||||
return commands[args.command](args)
|
||||
except (RunError, OSError) as exc:
|
||||
print(f"FAIL: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
429
okfwiki/readers.py
Normal file
429
okfwiki/readers.py
Normal file
|
|
@ -0,0 +1,429 @@
|
|||
"""A reader per file type: every unit of text in a document with its exact place. No model call (CLAUDE.md rule 1).
|
||||
|
||||
Usage:
|
||||
python3 -m okfwiki.readers locate FILE QUERIES
|
||||
|
||||
Places, as the file-type gate defines them (gates/filetypes.manifest.json): pdf {"page": N}, the physical page;
|
||||
docx {"paragraph": N}, the w:p elements of the body in order, empty ones and those in tables included, those in a text
|
||||
box or a compatibility fallback not; xlsx {"sheet": name, "cell": "B7"}; pptx {"slide": N, "notes": bool}, the slide
|
||||
list's order, hidden slides included; md, txt and html {"line": N}, for html the raw line of a visible text node
|
||||
(not in head, title, script, style, noscript, template or svg). Numbers are 1-based.
|
||||
|
||||
Hidden Office content (a hidden sheet, row or column, a hidden slide, hidden text, a tracked deletion) is kept apart
|
||||
from what a reader of the document sees. `locate` marks it hidden; `render` leaves it out and counts it.
|
||||
|
||||
Office files are read from their package XML with the standard library. PDF needs pypdf at exactly PDF_PIN, because
|
||||
the quote check needs the same raw text on every run (rule decisions R9 and R37); any other version is refused.
|
||||
|
||||
locate FILE and a JSON file {"file", "queries": [{"id", "quote"}]}: prints {"answers": [{"id", "found":
|
||||
[{"locator", "hidden"}]}]}, every place where the quote occurs, runs of whitespace collapsed to one space.
|
||||
Exit 0 = done, 2 = could not run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import html.parser
|
||||
import json
|
||||
import logging
|
||||
import posixpath
|
||||
import re
|
||||
import sys
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from xml.etree import ElementTree
|
||||
|
||||
PDF_PIN = "6.14.2"
|
||||
OFFICE = (".docx", ".xlsx", ".pptx")
|
||||
TYPES = (".pdf",) + OFFICE
|
||||
W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
|
||||
S = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
|
||||
P = "{http://schemas.openxmlformats.org/presentationml/2006/main}"
|
||||
A = "{http://schemas.openxmlformats.org/drawingml/2006/main}"
|
||||
R = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}"
|
||||
PKG = "{http://schemas.openxmlformats.org/package/2006/relationships}"
|
||||
MC_FALLBACK = "{http://schemas.openxmlformats.org/markup-compatibility/2006}Fallback"
|
||||
SPACE = re.compile(r"\s+")
|
||||
# A line of a rendering that Markdown would read as structure gets a backslash, so a paragraph is always a paragraph.
|
||||
STRUCTURE = re.compile(r"^(#|```|~~~|[-*+] |\d+[.)] |>|\||<)")
|
||||
# A sheet name heads its rows as their section, below the levels the skeleton proposes as concepts (title, 2 and 3):
|
||||
# "T1" or "Note" is a container, not a concept (trial of 07.11).
|
||||
SHEET_LEVEL = 4
|
||||
# A run of spreadsheet rows becomes one block until it holds about this many characters; each further block starts
|
||||
# with the run's first row again (most often its header), so a number far down a long table keeps its column names.
|
||||
# On the 22 real workbooks of the mixed set, one run without a limit held 1,190 rows and 1.07 million characters.
|
||||
ROW_BLOCK_CHARS = 4000
|
||||
STYLE_HEADING = re.compile(r"^heading ([1-6])$")
|
||||
|
||||
|
||||
class NoReader(Exception):
|
||||
"""The reader this file type needs is missing or not the pinned version."""
|
||||
|
||||
|
||||
class Unreadable(Exception):
|
||||
"""The file is not a sound file of its type."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Unit:
|
||||
locator: dict[str, Any]
|
||||
visible: str # what a reader of the document sees, whitespace collapsed
|
||||
text: str # visible plus hidden content, whitespace collapsed
|
||||
heading: int = 0 # the Markdown heading level the unit renders as, 0 for none
|
||||
group: str = "" # units of one group render as one block (a run of spreadsheet rows)
|
||||
line: str = "" # units of one line render as one line, each after its column letter (a spreadsheet row)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Rendering:
|
||||
text: str
|
||||
locators: list[dict[str, Any] | None] # the place of line N at index N-1, None for a blank line or a heading
|
||||
left_out: int # units with hidden content, left out of the text
|
||||
|
||||
|
||||
def collapse(text: str) -> str:
|
||||
return SPACE.sub(" ", text).strip()
|
||||
|
||||
|
||||
def xml(z: zipfile.ZipFile, part: str) -> ElementTree.Element:
|
||||
try:
|
||||
return ElementTree.fromstring(z.read(part))
|
||||
except KeyError as e:
|
||||
raise Unreadable(f"the package has no {part}") from e
|
||||
except ElementTree.ParseError as e:
|
||||
raise Unreadable(f"{part} is not XML: {e}") from e
|
||||
|
||||
|
||||
def targets(z: zipfile.ZipFile, part: str) -> dict[str, str]:
|
||||
"""Relationship id -> package part, for the relationships of part."""
|
||||
folder, name = posixpath.split(part)
|
||||
rels = posixpath.join(folder, "_rels", name + ".rels")
|
||||
if rels not in z.namelist():
|
||||
return {}
|
||||
found: dict[str, str] = {}
|
||||
for rel in xml(z, rels).iter(f"{PKG}Relationship"):
|
||||
target = rel.get("Target", "")
|
||||
if rel.get("TargetMode") == "External":
|
||||
continue
|
||||
found[rel.get("Id", "")] = (target[1:] if target.startswith("/")
|
||||
else posixpath.normpath(posixpath.join(folder, target)))
|
||||
return found
|
||||
|
||||
|
||||
def walk(node: ElementTree.Element, skip: tuple[str, ...]) -> list[ElementTree.Element]:
|
||||
"""node and its descendants in document order, without the subtrees of the tags in skip."""
|
||||
out = [node]
|
||||
for child in node:
|
||||
if child.tag not in skip:
|
||||
out += walk(child, skip)
|
||||
return out
|
||||
|
||||
|
||||
def docx_units(z: zipfile.ZipFile) -> list[Unit]:
|
||||
levels: dict[str, int] = {}
|
||||
if "word/styles.xml" in z.namelist():
|
||||
for style in xml(z, "word/styles.xml").iter(f"{W}style"):
|
||||
name = style.find(f"{W}name")
|
||||
m = STYLE_HEADING.match((name.get(f"{W}val", "") if name is not None else "").lower())
|
||||
if m or (name is not None and name.get(f"{W}val", "").lower() == "title"):
|
||||
levels[style.get(f"{W}styleId", "")] = int(m.group(1)) if m else 1
|
||||
body = xml(z, "word/document.xml").find(f"{W}body")
|
||||
if body is None:
|
||||
raise Unreadable("word/document.xml has no body")
|
||||
box = (f"{W}txbxContent", MC_FALLBACK)
|
||||
units: list[Unit] = []
|
||||
for p in (e for e in walk(body, box) if e.tag == f"{W}p"):
|
||||
visible: list[str] = []
|
||||
full: list[str] = []
|
||||
for run in (e for e in walk(p, box) if e.tag == f"{W}r"):
|
||||
props = run.find(f"{W}rPr")
|
||||
vanish = props.find(f"{W}vanish") if props is not None else None
|
||||
hidden = vanish is not None and vanish.get(f"{W}val", "true") not in ("0", "false")
|
||||
for e in run:
|
||||
piece = (e.text or "" if e.tag in (f"{W}t", f"{W}delText") else
|
||||
"\t" if e.tag == f"{W}tab" else " " if e.tag in (f"{W}br", f"{W}cr") else "")
|
||||
full.append(piece)
|
||||
if not hidden and e.tag != f"{W}delText":
|
||||
visible.append(piece)
|
||||
mark = p.find(f"{W}pPr/{W}pStyle")
|
||||
level = levels.get(mark.get(f"{W}val", "") if mark is not None else "", 0)
|
||||
units.append(Unit({"paragraph": len(units) + 1}, collapse("".join(visible)), collapse("".join(full)), level))
|
||||
return units
|
||||
|
||||
|
||||
def cell_text(cell: ElementTree.Element, shared: list[str]) -> str:
|
||||
kind = cell.get("t", "n")
|
||||
if kind == "inlineStr":
|
||||
node = cell.find(f"{S}is")
|
||||
return "" if node is None else string_item(node)
|
||||
value = cell.find(f"{S}v")
|
||||
raw = "" if value is None or value.text is None else value.text
|
||||
if kind == "s":
|
||||
try:
|
||||
return shared[int(raw)]
|
||||
except (ValueError, IndexError) as e:
|
||||
raise Unreadable(f"cell {cell.get('r')} names shared string {raw!r}, which does not exist") from e
|
||||
if kind == "b":
|
||||
return {"1": "TRUE", "0": "FALSE"}.get(raw, raw)
|
||||
return raw
|
||||
|
||||
|
||||
def string_item(si: ElementTree.Element) -> str:
|
||||
"""The text of a shared or inline string, rich runs joined, phonetic runs left out."""
|
||||
return "".join(t.text or "" for t in walk(si, (f"{S}rPh",)) if t.tag == f"{S}t")
|
||||
|
||||
|
||||
def letters(ref: str) -> str:
|
||||
"""The column letters of a cell reference: "AB" of "AB12"."""
|
||||
return re.match(r"[A-Za-z]*", ref).group(0) # type: ignore[union-attr]
|
||||
|
||||
|
||||
def column(ref: str) -> int:
|
||||
n = 0
|
||||
for ch in letters(ref).upper():
|
||||
n = n * 26 + ord(ch) - 64
|
||||
return n
|
||||
|
||||
|
||||
def xlsx_units(z: zipfile.ZipFile) -> list[Unit]:
|
||||
shared = ([string_item(si) for si in xml(z, "xl/sharedStrings.xml").iter(f"{S}si")]
|
||||
if "xl/sharedStrings.xml" in z.namelist() else [])
|
||||
parts = targets(z, "xl/workbook.xml")
|
||||
units: list[Unit] = []
|
||||
for sheet in xml(z, "xl/workbook.xml").iter(f"{S}sheet"):
|
||||
name = sheet.get("name", "")
|
||||
part = parts.get(sheet.get(f"{R}id", ""))
|
||||
if part is None:
|
||||
raise Unreadable(f"sheet {name!r} has no worksheet part")
|
||||
root = xml(z, part)
|
||||
sheet_hidden = sheet.get("state", "visible") != "visible"
|
||||
hidden_cols: set[int] = set()
|
||||
for col in root.iter(f"{S}col"):
|
||||
if col.get("hidden") in ("1", "true"):
|
||||
hidden_cols.update(range(int(col.get("min", "0")), int(col.get("max", "0")) + 1))
|
||||
first = True
|
||||
block, last = 0, 0 # an empty row between two rows ends a block; a hidden row does not
|
||||
for row in root.iter(f"{S}row"):
|
||||
row_hidden = row.get("hidden") in ("1", "true")
|
||||
number = int(row.get("r", "0") or "0")
|
||||
shown = not row_hidden and any(collapse(cell_text(c, shared)) for c in row.iter(f"{S}c"))
|
||||
if shown:
|
||||
if number != last + 1:
|
||||
block += 1
|
||||
last = number
|
||||
elif row_hidden and number == last + 1:
|
||||
last = number
|
||||
for cell in row.iter(f"{S}c"):
|
||||
ref = cell.get("r", "")
|
||||
text = collapse(cell_text(cell, shared))
|
||||
if not text:
|
||||
continue
|
||||
hidden = sheet_hidden or row_hidden or column(ref) in hidden_cols
|
||||
heading = name if first and not sheet_hidden else ""
|
||||
if heading:
|
||||
units.append(Unit({"sheet": name}, heading, heading, SHEET_LEVEL))
|
||||
first = False
|
||||
units.append(Unit({"sheet": name, "cell": ref}, "" if hidden else text, text, 0,
|
||||
f"{name}\n{block}", f"{name}\n{row.get('r', '')}"))
|
||||
return units
|
||||
|
||||
|
||||
def slide_paragraphs(root: ElementTree.Element) -> list[tuple[str, bool]]:
|
||||
"""(text, is the title) of every paragraph in a slide or notes part, in document order."""
|
||||
out: list[tuple[str, bool]] = []
|
||||
for shape in (e for e in walk(root, (MC_FALLBACK,)) if e.tag == f"{P}sp" or e.tag == f"{P}graphicFrame"):
|
||||
ph = shape.find(f"{P}nvSpPr/{P}nvPr/{P}ph")
|
||||
if ph is not None and ph.get("type") in ("sldNum", "sldImg", "dt", "ftr", "hdr"):
|
||||
continue
|
||||
title = ph is not None and ph.get("type") in ("title", "ctrTitle")
|
||||
for para in shape.iter(f"{A}p"):
|
||||
text = "".join((e.text or "") if e.tag == f"{A}t" else " " if e.tag == f"{A}br" else ""
|
||||
for e in para.iter())
|
||||
out.append((collapse(text), title))
|
||||
return out
|
||||
|
||||
|
||||
def pptx_units(z: zipfile.ZipFile) -> list[Unit]:
|
||||
parts = targets(z, "ppt/presentation.xml")
|
||||
units: list[Unit] = []
|
||||
for n, sld in enumerate(xml(z, "ppt/presentation.xml").iter(f"{P}sldId"), 1):
|
||||
part = parts.get(sld.get(f"{R}id", ""))
|
||||
if part is None:
|
||||
raise Unreadable(f"slide {n} has no part")
|
||||
root = xml(z, part)
|
||||
hidden = root.get("show") in ("0", "false")
|
||||
for text, title in slide_paragraphs(root):
|
||||
if text:
|
||||
units.append(Unit({"slide": n, "notes": False}, "" if hidden else text, text, 2 if title else 0))
|
||||
notes = [t for t in targets(z, part).values() if "/notesSlides/" in "/" + t]
|
||||
for note in notes[:1]:
|
||||
for text, _ in slide_paragraphs(xml(z, note)):
|
||||
if text:
|
||||
units.append(Unit({"slide": n, "notes": True}, "" if hidden else text, text))
|
||||
return units
|
||||
|
||||
|
||||
def pdf_units(path: Path) -> list[Unit]:
|
||||
try:
|
||||
import pypdf # type: ignore[import-not-found, unused-ignore]
|
||||
except ImportError as e:
|
||||
raise NoReader(f"{path.name}: reading PDF needs pypdf=={PDF_PIN} (python3 -m pip install pypdf=={PDF_PIN})") from e
|
||||
logging.getLogger("pypdf").setLevel(logging.ERROR) # repairs of a sloppy file are not the user's business
|
||||
if pypdf.__version__ != PDF_PIN:
|
||||
raise NoReader(f"{path.name}: reading PDF needs pypdf=={PDF_PIN}, found {pypdf.__version__} "
|
||||
f"(python3 -m pip install pypdf=={PDF_PIN})")
|
||||
try:
|
||||
doc = pypdf.PdfReader(str(path))
|
||||
if doc.is_encrypted and not doc.decrypt(""):
|
||||
raise Unreadable(f"{path.name} is encrypted")
|
||||
texts = [page.extract_text() or "" for page in doc.pages]
|
||||
except Unreadable:
|
||||
raise
|
||||
except Exception as e: # pypdf raises many kinds on a broken file
|
||||
raise Unreadable(f"{path.name} is not a readable PDF: {e}") from e
|
||||
return [Unit({"page": n}, collapse(t), collapse(t)) for n, t in enumerate(texts, 1)]
|
||||
|
||||
|
||||
INVISIBLE = frozenset({"head", "title", "script", "style", "noscript", "template", "svg"})
|
||||
|
||||
|
||||
class TextNodes(html.parser.HTMLParser):
|
||||
"""Every visible text node of an HTML page, cut at its line breaks: (line of the raw source, text)."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.open: list[str] = []
|
||||
self.found: list[tuple[int, str]] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if tag in INVISIBLE:
|
||||
self.open.append(tag)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag in self.open:
|
||||
del self.open[len(self.open) - 1 - self.open[::-1].index(tag):]
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if not self.open:
|
||||
line = self.getpos()[0]
|
||||
self.found += [(line + i, piece) for i, piece in enumerate(data.split("\n"))]
|
||||
|
||||
|
||||
def html_units(path: Path) -> list[Unit]:
|
||||
parser = TextNodes()
|
||||
try:
|
||||
parser.feed(path.read_text(encoding="utf-8"))
|
||||
parser.close()
|
||||
except UnicodeDecodeError as e:
|
||||
raise Unreadable(f"{path.name} is not UTF-8") from e
|
||||
return [Unit({"line": n}, collapse(t), collapse(t)) for n, t in parser.found if t.strip()]
|
||||
|
||||
|
||||
def text_units(path: Path) -> list[Unit]:
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8").split("\n")
|
||||
except UnicodeDecodeError as e:
|
||||
raise Unreadable(f"{path.name} is not UTF-8") from e
|
||||
return [Unit({"line": n}, collapse(t), collapse(t)) for n, t in enumerate(lines, 1)]
|
||||
|
||||
|
||||
def units(path: Path) -> list[Unit]:
|
||||
ext = path.suffix.lower()
|
||||
if ext == ".pdf":
|
||||
return pdf_units(path)
|
||||
if ext in (".html", ".htm"):
|
||||
return html_units(path)
|
||||
if ext not in OFFICE:
|
||||
return text_units(path)
|
||||
try:
|
||||
with zipfile.ZipFile(path) as z:
|
||||
return {".docx": docx_units, ".xlsx": xlsx_units, ".pptx": pptx_units}[ext](z)
|
||||
except zipfile.BadZipFile as e:
|
||||
raise Unreadable(f"{path.name} is not an Office package: {e}") from e
|
||||
|
||||
|
||||
def render(path: Path) -> Rendering:
|
||||
"""The text a reader of the document sees, as Markdown the block splitter reads: a heading line per heading unit,
|
||||
one line per unit, a blank line between blocks (a unit, or a run of spreadsheet rows). A spreadsheet row is one
|
||||
line, each cell after its column letter ("A: Quantity | C: 101000"), so a number keeps the column its header
|
||||
names; its place lists the cells with the offset each starts at (common.place_at). Hidden units are left out."""
|
||||
lines: list[str] = []
|
||||
places: list[dict[str, Any] | None] = []
|
||||
left_out = 0
|
||||
group: str | None = None
|
||||
line: str | None = None
|
||||
block, first_row = 0, 0 # where the current block, and the current run of rows, start in lines
|
||||
for u in units(path):
|
||||
if u.visible != u.text:
|
||||
left_out += 1
|
||||
if not u.visible:
|
||||
continue
|
||||
if u.line and u.line == line:
|
||||
lines[-1] += " | "
|
||||
places[-1]["cells"].append([u.locator["cell"], len(lines[-1])]) # type: ignore[index]
|
||||
lines[-1] += f"{letters(u.locator['cell'])}: {u.visible}"
|
||||
continue
|
||||
same_run = bool(u.group and u.group == group)
|
||||
if lines and not same_run:
|
||||
lines.append("")
|
||||
places.append(None)
|
||||
block = first_row = len(lines)
|
||||
elif same_run and u.line and sum(len(x) for x in lines[block:]) >= ROW_BLOCK_CHARS:
|
||||
head, at = lines[first_row], places[first_row] or {}
|
||||
lines += ["", head]
|
||||
places += [None, {"sheet": at.get("sheet"), "cells": [list(c) for c in at.get("cells", [])]}]
|
||||
block = len(lines) - 1
|
||||
group, line = u.group or None, u.line or None
|
||||
if u.line:
|
||||
lines.append(f"{letters(u.locator['cell'])}: {u.visible}")
|
||||
places.append({"sheet": u.locator["sheet"], "cells": [[u.locator["cell"], 0]]})
|
||||
elif u.heading:
|
||||
lines.append("#" * u.heading + " " + u.visible)
|
||||
places.append(None)
|
||||
else:
|
||||
lines.append(("\\" + u.visible) if STRUCTURE.match(u.visible) else u.visible)
|
||||
places.append(u.locator)
|
||||
return Rendering("\n".join(lines) + "\n" if lines else "", places, left_out)
|
||||
|
||||
|
||||
def locate(path: Path, quotes: list[str]) -> list[list[tuple[dict[str, Any], bool]]]:
|
||||
"""For each quote, every (place, hidden) where it occurs, in the order of the document."""
|
||||
found = units(path)
|
||||
out: list[list[tuple[dict[str, Any], bool]]] = []
|
||||
for quote in quotes:
|
||||
q = collapse(quote)
|
||||
places: list[tuple[dict[str, Any], bool]] = []
|
||||
for u in found:
|
||||
if "cell" not in u.locator and "sheet" in u.locator:
|
||||
continue # a sheet's own heading unit is not a place in the workbook
|
||||
if q and q in u.visible:
|
||||
places.append((u.locator, False))
|
||||
elif q and q in u.text:
|
||||
places.append((u.locator, True))
|
||||
out.append(places)
|
||||
return out
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="python3 -m okfwiki.readers", description=__doc__.split("\n")[0])
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
p = sub.add_parser("locate")
|
||||
p.add_argument("file", type=Path)
|
||||
p.add_argument("queries", type=Path)
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
doc = json.loads(args.queries.read_text(encoding="utf-8"))
|
||||
queries = [(q["id"], q["quote"]) for q in doc["queries"]]
|
||||
answers = locate(args.file, [quote for _, quote in queries])
|
||||
except (OSError, ValueError, KeyError, TypeError, NoReader, Unreadable) as e:
|
||||
print(f"readers cannot run: {e}", file=sys.stderr)
|
||||
return 2
|
||||
print(json.dumps({"answers": [{"id": i, "found": [{"locator": loc, "hidden": h} for loc, h in places]}
|
||||
for (i, _), places in zip(queries, answers)]}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
569
okfwiki/serve.py
Normal file
569
okfwiki/serve.py
Normal file
|
|
@ -0,0 +1,569 @@
|
|||
"""A read-only MCP server over one or more built collections (motor C21). It calls no model and writes no file.
|
||||
|
||||
serve --bundle NAME WIKI [SOURCES] [--bundle NAME WIKI [SOURCES] ...]
|
||||
|
||||
Speaks the MCP stdio transport: JSON-RPC 2.0, one message per line on stdin and stdout; anything else goes to stderr.
|
||||
Register it with Claude Code as, for example,
|
||||
`claude mcp add okf-wiki -- okf-wiki serve --bundle hage /path/wiki /path/sources`. Four tools, the skill's router for
|
||||
answering a question:
|
||||
|
||||
collections each collection: its folders, pages, main pages, source files and okf_version
|
||||
search words against each page's title, other names and description in index.md (SPEC section 8) and its
|
||||
text (footnote lines and link targets left out); a page ranks by how rare its matching words are, a
|
||||
word in the title or in another name of the page (`x-aliases`, at the end of its index line as
|
||||
" (also: A; B)") counting twice;
|
||||
a word matches the same word or one with an ending of at most two letters on a stem of four
|
||||
(kmerge.word_match); main_only keeps main pages, and with no words lists them all, one per document
|
||||
page one concept by its id (SPEC section 2: its path without .md): frontmatter, other names, sources with
|
||||
quote and place, body, links out (each marked when it names no page) and the pages that link to it
|
||||
source lines of the frozen copy of a source around a cited line, with the heading above, each line's page,
|
||||
slide or cell, and the original file
|
||||
|
||||
Titles and descriptions come from index.md, and page text, main pages and inbound links from a plain scan of the
|
||||
page files without parsing their YAML (which takes 67 s on wiki-38): the first call over wiki-38, 3 141 pages, reads
|
||||
it in 2.3 s (08.11), and a search after that takes milliseconds. A collection is read again when its index.md or its
|
||||
folder changes. Nothing outside a collection's folder or its copy of the sources is read. Plain file search in the
|
||||
same folders works beside it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, TextIO
|
||||
|
||||
from okfwiki.audit import LINK, SCHEME, read_document, resolve
|
||||
from okfwiki.common import place_words
|
||||
from okfwiki.kmerge import name_words, word_match
|
||||
from okfwiki.view import nearest_heading
|
||||
|
||||
PROTOCOL_VERSIONS = ("2025-11-25", "2025-06-18", "2025-03-26", "2024-11-05") # newest first
|
||||
SERVER = {"name": "okf-wiki", "version": "1"}
|
||||
RESERVED = ("index.md", "log.md") # SPEC 3.1
|
||||
FOOTNOTE_DEF = re.compile(r"^\[\^[^\]]+\]:")
|
||||
INDEX_LINE = re.compile(r"^[-*] \[(?P<title>.*?)\]\((?P<href>[^()\s]+?\.md)\)(?: - (?P<desc>.*))?$")
|
||||
ALSO = re.compile(r"^(?P<desc>.*) \(also: (?P<names>.+)\)$") # a page's other names, as kassemble ends its index line
|
||||
DEFAULT_LIMIT = 20
|
||||
DEFAULT_WINDOW = 2
|
||||
# Claude Code (2.1.319, 08.11) kept a tool result of 47 062 characters whole and put one of 55 819 in a file; a page
|
||||
# longer than this comes in parts, each asked for with `part`.
|
||||
PART_CHARS = 40_000
|
||||
|
||||
INSTRUCTIONS = (
|
||||
"Read-only lookups in built okf-wiki collections. No call here runs a model, and nothing is written.\n"
|
||||
"To answer a question: (1) `search` each collection's pages with the question's words, in the collection's own "
|
||||
"words and language: it matches words, not meaning, and says how many pages each word hits and how many pages "
|
||||
"matched of how many; (2) open the main page of the area with `page` (`search` with main_only lists the main "
|
||||
"pages, one per document); (3) follow its links_out with `page`, and linked_from to see what builds on it; "
|
||||
"(4) open the source at a footnote's line with `source` when the quote has to be checked or the page does not "
|
||||
"say enough. Cite the page and the source (document and place). Say plainly what the collections do not cover. "
|
||||
"Plain file search in the collection folders works beside this server.")
|
||||
|
||||
|
||||
class ToolError(Exception):
|
||||
"""A call the server cannot answer: reported to the caller as a tool result with isError."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Collection:
|
||||
name: str
|
||||
wiki: Path
|
||||
sources: Path | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Entry:
|
||||
id: str
|
||||
title: str
|
||||
description: str
|
||||
title_words: set[str] # of the title and of the page's other names
|
||||
words: set[str] # title, other names, description and page text
|
||||
aliases: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Catalogue:
|
||||
stamp: tuple[int, int, int]
|
||||
okf_version: str | None
|
||||
entries: list[Entry]
|
||||
main_for: dict[str, list[str]]
|
||||
linked_from: dict[str, list[str]]
|
||||
vocabulary: dict[str, set[int]] = field(default_factory=dict) # word -> entry indexes
|
||||
|
||||
|
||||
# --- reading a collection ----------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def stamp(wiki: Path) -> tuple[int, int, int]:
|
||||
index = (wiki / "index.md").stat()
|
||||
return index.st_mtime_ns, index.st_size, wiki.stat().st_mtime_ns
|
||||
|
||||
|
||||
def frontmatter_text(text: str) -> tuple[str, str]:
|
||||
"""(frontmatter, body) of a page as plain text, split at the closing --- (SPEC 4), without parsing the YAML."""
|
||||
lines = text.split("\n")
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return "", text
|
||||
end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
|
||||
if end is None:
|
||||
return "", text
|
||||
return "\n".join(lines[1:end]), "\n".join(lines[end + 1:])
|
||||
|
||||
|
||||
def main_for_of(front: str) -> list[str]:
|
||||
"""The sources a page is the main page of (`x-main-for`), read from the frontmatter text: a block list as
|
||||
kassemble writes it, or one value on the key's line."""
|
||||
out: list[str] = []
|
||||
lines = front.split("\n")
|
||||
for i, raw in enumerate(lines):
|
||||
if not raw.startswith("x-main-for:"):
|
||||
continue
|
||||
rest = raw[len("x-main-for:"):].strip()
|
||||
if rest:
|
||||
out.extend(v.strip().strip("'\"") for v in rest.strip("[]").split(",") if v.strip())
|
||||
for item in lines[i + 1:]:
|
||||
if not item.startswith("- "):
|
||||
break
|
||||
out.append(item[2:].strip().strip("'\""))
|
||||
return out
|
||||
|
||||
|
||||
def concept_id(rel: str) -> str:
|
||||
return rel[:-3] if rel.endswith(".md") else rel
|
||||
|
||||
|
||||
def load_catalogue(wiki: Path) -> Catalogue:
|
||||
st = stamp(wiki)
|
||||
index_text = (wiki / "index.md").read_text(encoding="utf-8")
|
||||
front, body = frontmatter_text(index_text)
|
||||
version = next((ln.split(":", 1)[1].strip().strip("'\"") for ln in front.split("\n")
|
||||
if ln.startswith("okf_version:")), None)
|
||||
entries: list[Entry] = []
|
||||
seen: set[str] = set()
|
||||
for raw in body.split("\n"):
|
||||
m = INDEX_LINE.match(raw.strip())
|
||||
if not m:
|
||||
continue
|
||||
rel = resolve("index.md", m.group("href"))
|
||||
if not rel or rel in seen or rel.split("/")[-1] in RESERVED:
|
||||
continue
|
||||
seen.add(rel)
|
||||
title, desc = m.group("title"), m.group("desc") or ""
|
||||
also = ALSO.match(desc)
|
||||
aliases = [a.strip() for a in also.group("names").split("; ") if a.strip()] if also else []
|
||||
desc = also.group("desc") if also else desc
|
||||
tw = set(name_words(" ".join([title, *aliases])))
|
||||
entries.append(Entry(concept_id(rel), title, desc, tw, tw | set(name_words(desc)), aliases))
|
||||
main_for: dict[str, list[str]] = {}
|
||||
linked: dict[str, set[str]] = {}
|
||||
body_words: dict[str, set[str]] = {}
|
||||
for p in sorted(wiki.rglob("*.md")):
|
||||
rel = p.relative_to(wiki).as_posix()
|
||||
if p.name in RESERVED or not p.is_file():
|
||||
continue
|
||||
try:
|
||||
page_front, page_body = frontmatter_text(p.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
mains = main_for_of(page_front)
|
||||
if mains:
|
||||
main_for[concept_id(rel)] = mains
|
||||
text = "\n".join(ln for ln in page_body.split("\n") if not FOOTNOTE_DEF.match(ln))
|
||||
body_words[concept_id(rel)] = set(name_words(LINK.sub(lambda m: m.group(1), text)))
|
||||
for m in LINK.finditer(page_body):
|
||||
target = m.group(2)
|
||||
if not target or target.startswith("#") or SCHEME.match(target):
|
||||
continue
|
||||
to = resolve(rel, target)
|
||||
if to and to.endswith(".md") and to != rel:
|
||||
linked.setdefault(concept_id(to), set()).add(concept_id(rel))
|
||||
cat = Catalogue(st, version, entries, main_for, {k: sorted(v) for k, v in linked.items()})
|
||||
for n, e in enumerate(entries):
|
||||
e.words |= body_words.get(e.id, set())
|
||||
for w in e.words:
|
||||
cat.vocabulary.setdefault(w, set()).add(n)
|
||||
return cat
|
||||
|
||||
|
||||
# --- the server --------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def schema(props: dict[str, Any], required: list[str]) -> dict[str, Any]:
|
||||
return {"type": "object", "properties": props, "required": required, "additionalProperties": False}
|
||||
|
||||
|
||||
COLLECTION_ARG = {"type": "string", "description": "The collection's name, as `collections` lists it."}
|
||||
TOOLS: list[dict[str, Any]] = [
|
||||
{"name": "collections",
|
||||
"description": "List the collections this server reads: name, folders, pages, main pages, source files.",
|
||||
"inputSchema": schema({}, [])},
|
||||
{"name": "search",
|
||||
"description": ("Search the pages of one or all collections (title, other names, description and text) for words. "
|
||||
"Matches words, not meaning: use the collection's own words and language, and ask again with "
|
||||
"other words when the counts say a word hits nothing. main_only keeps main pages (one per "
|
||||
"document); main_only with no words lists them all."),
|
||||
"inputSchema": schema({
|
||||
"words": {"type": "string", "description": "The words to look for, separated by spaces."},
|
||||
"collection": COLLECTION_ARG,
|
||||
"main_only": {"type": "boolean", "description": "Only main pages (default false)."},
|
||||
"limit": {"type": "integer", "minimum": 1, "description": f"Results per collection (default {DEFAULT_LIMIT})."},
|
||||
"offset": {"type": "integer", "minimum": 0, "description": "Results to skip, for the next page (default 0)."},
|
||||
}, [])},
|
||||
{"name": "page",
|
||||
"description": ("One concept page by its id (its file name without .md): title, description, sources with the "
|
||||
"verbatim quote and its place, the body with footnotes, links out and the pages linking to it. "
|
||||
"A long page comes in parts: `parts` says how many, ask for the next with `part`."),
|
||||
"inputSchema": schema({"collection": COLLECTION_ARG,
|
||||
"id": {"type": "string", "description": "The concept id, e.g. from search."},
|
||||
"part": {"type": "integer", "minimum": 1, "description": "Which part (default 1)."}},
|
||||
["collection", "id"])},
|
||||
{"name": "source",
|
||||
"description": ("Lines of a source's frozen copy around a cited line, with the heading above, each line's page, "
|
||||
"slide or cell, and the original file. resource and line are a page source's resource and line."),
|
||||
"inputSchema": schema({
|
||||
"collection": COLLECTION_ARG,
|
||||
"resource": {"type": "string", "description": "The source's resource, e.g. /kilde/report.pdf.md."},
|
||||
"line": {"type": "integer", "minimum": 1, "description": "The cited line (1-based)."},
|
||||
"before": {"type": "integer", "minimum": 0, "description": f"Lines before it (default {DEFAULT_WINDOW})."},
|
||||
"after": {"type": "integer", "minimum": 0, "description": f"Lines after it (default {DEFAULT_WINDOW})."},
|
||||
}, ["collection", "resource", "line"])},
|
||||
]
|
||||
for _tool in TOOLS:
|
||||
_tool["annotations"] = {"readOnlyHint": True, "destructiveHint": False, "idempotentHint": True,
|
||||
"openWorldHint": False}
|
||||
|
||||
|
||||
def int_arg(args: dict[str, Any], key: str, default: int, least: int) -> int:
|
||||
value = args.get(key, default)
|
||||
if type(value) is not int or value < least:
|
||||
raise ToolError(f"{key} must be a whole number of at least {least}")
|
||||
return int(value)
|
||||
|
||||
|
||||
class Server:
|
||||
def __init__(self, collections: list[Collection]) -> None:
|
||||
self.collections = {c.name: c for c in collections}
|
||||
self.catalogues: dict[str, Catalogue] = {}
|
||||
|
||||
# protocol
|
||||
|
||||
def handle(self, msg: Any) -> dict[str, Any] | None:
|
||||
if not isinstance(msg, dict) or msg.get("jsonrpc") != "2.0" or not isinstance(msg.get("method"), str):
|
||||
return error(msg.get("id") if isinstance(msg, dict) else None, -32600, "not a JSON-RPC 2.0 request")
|
||||
if "id" not in msg:
|
||||
return None # a notification
|
||||
mid, method, params = msg["id"], msg["method"], msg.get("params") or {}
|
||||
if method == "initialize":
|
||||
asked = params.get("protocolVersion") if isinstance(params, dict) else None
|
||||
version = asked if asked in PROTOCOL_VERSIONS else PROTOCOL_VERSIONS[0]
|
||||
return reply(mid, {"protocolVersion": version, "capabilities": {"tools": {"listChanged": False}},
|
||||
"serverInfo": SERVER, "instructions": INSTRUCTIONS})
|
||||
if method == "ping":
|
||||
return reply(mid, {})
|
||||
if method == "tools/list":
|
||||
return reply(mid, {"tools": TOOLS})
|
||||
if method == "tools/call":
|
||||
name = params.get("name") if isinstance(params, dict) else None
|
||||
args = params.get("arguments") or {} if isinstance(params, dict) else {}
|
||||
tool = getattr(self, f"tool_{name}", None) if name in {t["name"] for t in TOOLS} else None
|
||||
if tool is None or not isinstance(args, dict):
|
||||
return error(mid, -32602, f"unknown tool {name!r}" if tool is None else "arguments must be an object")
|
||||
try:
|
||||
data, is_error = tool(args), False
|
||||
except ToolError as exc:
|
||||
data, is_error = {"error": str(exc)}, True
|
||||
text = dumps(data)
|
||||
return reply(mid, {"content": [{"type": "text", "text": text}], "isError": is_error})
|
||||
return error(mid, -32601, f"method not found: {method}")
|
||||
|
||||
# collections
|
||||
|
||||
def collection(self, name: Any) -> Collection:
|
||||
if name not in self.collections:
|
||||
raise ToolError(f"no collection {name!r}; this server reads {', '.join(sorted(self.collections))}")
|
||||
return self.collections[str(name)]
|
||||
|
||||
def catalogue(self, c: Collection) -> Catalogue:
|
||||
try:
|
||||
current = stamp(c.wiki)
|
||||
cat = self.catalogues.get(c.name)
|
||||
if cat is None or cat.stamp != current:
|
||||
cat = self.catalogues[c.name] = load_catalogue(c.wiki)
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
raise ToolError(f"collection {c.name} cannot be read: {exc}") from exc
|
||||
return cat
|
||||
|
||||
def tool_collections(self, args: dict[str, Any]) -> dict[str, Any]:
|
||||
out = []
|
||||
for c in self.collections.values():
|
||||
cat = self.catalogue(c)
|
||||
files = None
|
||||
if c.sources is not None:
|
||||
files = sum(1 for p in (c.sources / "kilde").rglob("*") if p.is_file())
|
||||
out.append({"collection": c.name, "wiki": str(c.wiki.resolve()),
|
||||
"sources": str(c.sources.resolve()) if c.sources else None,
|
||||
"pages": len(cat.entries), "main_pages": len(cat.main_for), "source_files": files,
|
||||
"okf_version": cat.okf_version})
|
||||
return {"collections": out}
|
||||
|
||||
# search
|
||||
|
||||
def tool_search(self, args: dict[str, Any]) -> dict[str, Any]:
|
||||
words = args.get("words", "")
|
||||
if not isinstance(words, str):
|
||||
raise ToolError("words must be a string")
|
||||
main_only = args.get("main_only", False) is True
|
||||
limit = int_arg(args, "limit", DEFAULT_LIMIT, 1)
|
||||
offset = int_arg(args, "offset", 0, 0)
|
||||
asked = list(dict.fromkeys(name_words(words)))
|
||||
if not asked and not main_only:
|
||||
raise ToolError("give words to search for, or main_only to list the main pages")
|
||||
chosen = [self.collection(args["collection"])] if "collection" in args else list(self.collections.values())
|
||||
return {"collections": [self.search_one(c, asked, main_only, limit, offset) for c in chosen]}
|
||||
|
||||
def search_one(self, c: Collection, asked: list[str], main_only: bool, limit: int, offset: int) -> dict[str, Any]:
|
||||
cat = self.catalogue(c)
|
||||
n = len(cat.entries)
|
||||
hits: dict[int, list[str]] = {}
|
||||
counts = []
|
||||
weight: dict[str, float] = {}
|
||||
for word in asked:
|
||||
found: set[int] = set()
|
||||
for known, at in cat.vocabulary.items():
|
||||
if word_match(word, known):
|
||||
found |= at
|
||||
counts.append({"word": word, "pages": len(found)})
|
||||
weight[word] = math.log((n + 1) / (len(found) + 0.5))
|
||||
for i in found:
|
||||
hits.setdefault(i, []).append(word)
|
||||
if asked:
|
||||
pool = list(hits)
|
||||
else:
|
||||
pool = list(range(n))
|
||||
results = []
|
||||
for i in pool:
|
||||
e = cat.entries[i]
|
||||
if main_only and e.id not in cat.main_for:
|
||||
continue
|
||||
matched = hits.get(i, [])
|
||||
score = sum(weight[w] * (2 if any(word_match(w, t) for t in e.title_words) else 1) for w in matched)
|
||||
results.append((-score, e.title.casefold(), e.id, e, matched, score))
|
||||
results.sort(key=lambda r: (r[0], r[1], r[2]))
|
||||
page = results[offset:offset + limit]
|
||||
return {"collection": c.name, "pages": n, "matched": len(results), "words": counts, "offset": offset,
|
||||
"more": offset + limit < len(results),
|
||||
"results": [{"id": e.id, "title": e.title, "aliases": e.aliases, "description": e.description,
|
||||
"main_for": cat.main_for.get(e.id, []), "score": round(score, 3), "words": matched}
|
||||
for _, _, _, e, matched, score in page]}
|
||||
|
||||
# page
|
||||
|
||||
def page_rel(self, c: Collection, pid: Any) -> str:
|
||||
if not isinstance(pid, str) or not pid.strip():
|
||||
raise ToolError("id must be a concept id, e.g. from search")
|
||||
rel = pid.strip()
|
||||
rel = rel if rel.endswith(".md") else rel + ".md"
|
||||
norm = resolve("index.md", rel.lstrip("/")) if not rel.startswith("/") else None
|
||||
path = (c.wiki / norm) if norm else None
|
||||
if (not norm or norm.split("/")[-1] in RESERVED or path is None or not path.is_file()
|
||||
or not path.resolve().is_relative_to(c.wiki.resolve())):
|
||||
raise ToolError(f"no concept {pid!r} in {c.name}; find its id with search")
|
||||
return norm
|
||||
|
||||
def tool_page(self, args: dict[str, Any]) -> dict[str, Any]:
|
||||
c = self.collection(args.get("collection"))
|
||||
cat = self.catalogue(c)
|
||||
rel = self.page_rel(c, args.get("id"))
|
||||
try:
|
||||
doc = read_document(c.wiki / rel, rel)
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
raise ToolError(f"{rel} cannot be read: {exc}") from exc
|
||||
fm = doc.frontmatter or {}
|
||||
srcs = fm.get("sources", [])
|
||||
sources = []
|
||||
for s in srcs if isinstance(srcs, list) else [srcs]:
|
||||
if not isinstance(s, dict):
|
||||
continue
|
||||
raw_loc = s.get("x-locator")
|
||||
loc: dict[str, Any] = raw_loc if isinstance(raw_loc, dict) else {}
|
||||
sources.append({"id": s.get("id"), "resource": s.get("resource"), "title": s.get("title"),
|
||||
"line": loc.get("line"), "place": place_words(loc) if loc else None,
|
||||
"quote": s.get("x-quote"), "provenance": s.get("x-provenance")})
|
||||
out_links: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for m in LINK.finditer(doc.body):
|
||||
target = m.group(2)
|
||||
if not target or target.startswith("#") or SCHEME.match(target):
|
||||
continue
|
||||
to = resolve(rel, target)
|
||||
if not to or not to.endswith(".md") or to == rel or to in seen:
|
||||
continue
|
||||
seen.add(to)
|
||||
out_links.append({"id": concept_id(to), "exists": (c.wiki / to).is_file()})
|
||||
mains = fm.get("x-main-for", [])
|
||||
names = fm.get("x-aliases", [])
|
||||
head = {"collection": c.name, "id": concept_id(rel), "path": str((c.wiki / rel).resolve()),
|
||||
"title": fm.get("title"), "aliases": names if isinstance(names, list) else [names],
|
||||
"description": fm.get("description"), "type": fm.get("type"),
|
||||
"main_for": mains if isinstance(mains, list) else [mains],
|
||||
"links_out": out_links, "linked_from": cat.linked_from.get(concept_id(rel), [])}
|
||||
parts = split_parts(head, doc.body.strip(), sources)
|
||||
part = int_arg(args, "part", 1, 1)
|
||||
if part > len(parts):
|
||||
raise ToolError(f"{concept_id(rel)} has {len(parts)} part(s), not {part}")
|
||||
return parts[part - 1]
|
||||
|
||||
# source
|
||||
|
||||
def tool_source(self, args: dict[str, Any]) -> dict[str, Any]:
|
||||
c = self.collection(args.get("collection"))
|
||||
if c.sources is None:
|
||||
raise ToolError(f"collection {c.name} was started without its sources folder; give it as the third "
|
||||
f"value of --bundle")
|
||||
res = args.get("resource")
|
||||
line = int_arg(args, "line", 0, 1)
|
||||
before = int_arg(args, "before", DEFAULT_WINDOW, 0)
|
||||
after = int_arg(args, "after", DEFAULT_WINDOW, 0)
|
||||
rel = resolve("index.md", res) if isinstance(res, str) and res and not SCHEME.match(res) else None
|
||||
root = c.sources.resolve()
|
||||
path = (c.sources / rel) if rel else None
|
||||
if (not rel or not rel.startswith("kilde/") or path is None or not path.is_file()
|
||||
or not path.resolve().is_relative_to(root)):
|
||||
raise ToolError(f"no source {res!r} in the sources of {c.name}; use a resource a page's sources name")
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
raise ToolError(f"{rel} cannot be read: {exc}") from exc
|
||||
if line > len(lines):
|
||||
raise ToolError(f"{res} has {len(lines)} lines, not {line}")
|
||||
entry = self.manifest_entry(c, rel)
|
||||
places = entry.get("places") if isinstance(entry.get("places"), list) else None
|
||||
shown = []
|
||||
for n in range(max(1, line - before), min(len(lines), line + after) + 1):
|
||||
item: dict[str, Any] = {"line": n, "text": lines[n - 1]}
|
||||
place = places[n - 1] if places is not None and n - 1 < len(places) else None
|
||||
if isinstance(place, dict):
|
||||
item["place"] = place_words(place)
|
||||
shown.append(item)
|
||||
heading = nearest_heading(lines, line)
|
||||
out: dict[str, Any] = {"collection": c.name, "resource": res, "path": str(path.resolve()),
|
||||
"lines_total": len(lines), "lines": shown,
|
||||
"heading": {"line": heading[0], "text": heading[1]} if heading else None}
|
||||
if isinstance(entry.get("original"), str):
|
||||
original: dict[str, Any] = {"name": Path(entry["original"]).name}
|
||||
if isinstance(entry.get("url"), str):
|
||||
original["url"] = entry["url"]
|
||||
copy = entry.get("original_copy")
|
||||
if isinstance(copy, str) and (c.sources / copy).resolve().is_relative_to(root):
|
||||
original["copy"] = str((c.sources / copy).resolve())
|
||||
out["original"] = original
|
||||
elif isinstance(entry.get("url"), str):
|
||||
out["original"] = {"url": entry["url"]}
|
||||
return out
|
||||
|
||||
def manifest_entry(self, c: Collection, rel: str) -> dict[str, Any]:
|
||||
assert c.sources is not None
|
||||
try:
|
||||
doc = json.loads((c.sources / "source-manifest.json").read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
for e in doc.get("sources", []) if isinstance(doc, dict) else []:
|
||||
if isinstance(e, dict) and e.get("path") == rel:
|
||||
return e
|
||||
return {}
|
||||
|
||||
|
||||
def dumps(data: Any) -> str:
|
||||
return json.dumps(data, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def split_parts(head: dict[str, Any], body: str, sources: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""A page as parts of at most PART_CHARS characters of JSON each: the first carries the page's fields, every part
|
||||
its share of the body (whole paragraphs, in order) and then of the sources (whole entries, in order). The bodies
|
||||
joined with a blank line are the page's body, and nothing is left out; one paragraph or entry longer than a part
|
||||
stands alone in its part."""
|
||||
items: list[tuple[str, Any]] = [("p", p) for p in body.split("\n\n")] if body else []
|
||||
items += [("s", s) for s in sources]
|
||||
|
||||
def build(base: dict[str, Any], chunk: list[tuple[str, Any]], k: int, n: int) -> dict[str, Any]:
|
||||
return {**base, "part": k, "parts": n, "body": "\n\n".join(v for t, v in chunk if t == "p"),
|
||||
"sources": [v for t, v in chunk if t == "s"]}
|
||||
|
||||
chunks: list[list[tuple[str, Any]]] = [[]]
|
||||
for item in items:
|
||||
base = head if len(chunks) == 1 else {"collection": head["collection"], "id": head["id"]}
|
||||
trial = chunks[-1] + [item]
|
||||
if chunks[-1] and len(dumps(build(base, trial, 99999, 99999))) > PART_CHARS:
|
||||
chunks.append([item])
|
||||
else:
|
||||
chunks[-1] = trial
|
||||
n = len(chunks)
|
||||
return [build(head if k == 1 else {"collection": head["collection"], "id": head["id"]}, chunk, k, n)
|
||||
for k, chunk in enumerate(chunks, 1)]
|
||||
|
||||
|
||||
def reply(mid: Any, result: dict[str, Any]) -> dict[str, Any]:
|
||||
return {"jsonrpc": "2.0", "id": mid, "result": result}
|
||||
|
||||
|
||||
def error(mid: Any, code: int, message: str) -> dict[str, Any]:
|
||||
return {"jsonrpc": "2.0", "id": mid, "error": {"code": code, "message": message}}
|
||||
|
||||
|
||||
def run(server: Server, inp: TextIO, out: TextIO) -> None:
|
||||
for raw in inp:
|
||||
if not raw.strip():
|
||||
continue
|
||||
try:
|
||||
msg = json.loads(raw)
|
||||
except ValueError:
|
||||
answer: dict[str, Any] | None = error(None, -32700, "not JSON")
|
||||
else:
|
||||
answer = server.handle(msg)
|
||||
if answer is not None:
|
||||
out.write(json.dumps(answer, ensure_ascii=False) + "\n")
|
||||
out.flush()
|
||||
|
||||
|
||||
def collections_from(groups: list[list[str]]) -> list[Collection]:
|
||||
out: list[Collection] = []
|
||||
for g in groups:
|
||||
if len(g) not in (2, 3):
|
||||
raise ValueError(f"--bundle takes NAME WIKI [SOURCES], not {' '.join(g)}")
|
||||
name, wiki = g[0], Path(g[1]).expanduser()
|
||||
sources = Path(g[2]).expanduser() if len(g) == 3 else None
|
||||
if any(c.name == name for c in out):
|
||||
raise ValueError(f"two collections are named {name}")
|
||||
if not (wiki / "index.md").is_file():
|
||||
raise ValueError(f"{wiki} has no index.md, so it is not a built collection")
|
||||
if sources is not None and not (sources / "source-manifest.json").is_file():
|
||||
raise ValueError(f"{sources} has no source-manifest.json, so it is not a sources folder")
|
||||
out.append(Collection(name, wiki, sources))
|
||||
return out
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
ap = argparse.ArgumentParser(prog="okf-wiki serve", description=__doc__.split("\n")[0])
|
||||
ap.add_argument("--bundle", action="append", nargs="+", required=True, metavar="NAME WIKI [SOURCES]",
|
||||
help="a collection to serve: its name, its folder, and its sources folder (repeat for more)")
|
||||
try:
|
||||
ns = ap.parse_args(argv)
|
||||
except SystemExit as exc:
|
||||
return int(exc.code or 0) if isinstance(exc.code, int) else 2
|
||||
try:
|
||||
collections = collections_from(ns.bundle)
|
||||
except ValueError as exc:
|
||||
print(f"serve: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
print(f"okf-wiki serve: {', '.join(c.name for c in collections)}", file=sys.stderr)
|
||||
run(Server(collections), sys.stdin, sys.stdout)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
830
okfwiki/sources.py
Normal file
830
okfwiki/sources.py
Normal file
|
|
@ -0,0 +1,830 @@
|
|||
"""The source manifest of a frozen copy (contract K1): check, freeze and subset; the change warning (contract K4): diff
|
||||
and marks. No network, no model.
|
||||
|
||||
Usage:
|
||||
python3 -m okfwiki.sources check ROOT
|
||||
python3 -m okfwiki.sources freeze STAGING --meta FILE --out ROOT [--frozen-at YYYY-MM-DDTHH:MM:SSZ]
|
||||
python3 -m okfwiki.sources folder DIR --collection NAME --out ROOT [--frozen-at YYYY-MM-DDTHH:MM:SSZ]
|
||||
python3 -m okfwiki.sources subset ROOT --paths FILE --out ROOT2 [--frozen-at YYYY-MM-DDTHH:MM:SSZ]
|
||||
python3 -m okfwiki.sources diff OLD NEW
|
||||
python3 -m okfwiki.sources marks COLLECTION --diff FILE
|
||||
python3 -m okfwiki.sources reread ROOT --out ROOT2 [--frozen-at YYYY-MM-DDTHH:MM:SSZ]
|
||||
python3 -m okfwiki.sources watch ROOT --into DIR [--bundle COLLECTION] [--frozen-at YYYY-MM-DDTHH:MM:SSZ]
|
||||
|
||||
A frozen copy is a root folder with source-manifest.json and the source files under kilde/. A page cites a source as
|
||||
sources[].resource "/" + path.
|
||||
|
||||
check Exit 0 when every listed file exists with the same sha256, bytes and lines, every file under kilde/ is
|
||||
listed, and the manifest has the form of K1; exit 1 with the problems listed; exit 2 when it cannot run (no
|
||||
manifest, not JSON, another schema, sources not a list).
|
||||
freeze Copies the files of STAGING (flat, named as the part of path after kilde/) to ROOT/kilde/, makes them read
|
||||
only and writes the manifest. The meta file says per file what freeze cannot know:
|
||||
{"collection": "...", "index": {...} (optional), "schema": "okf-wiki/source-meta/1",
|
||||
"sources": {"<name>": {"kind": "...", "main_topic": true|false, "url": "...", "version": "..."}}}
|
||||
Refused with exit 1 and the files listed: a file the meta lacks or a meta entry without a file, an invalid
|
||||
entry, a file that is not UTF-8, and a file whose lines differ between splitlines() and split("\\n") (the
|
||||
auditor uses the one, the trial code the other, so line N must be the same for both). Exit 2 when ROOT exists.
|
||||
folder Freezes a folder of documents as it is, with no staging and no hand-written meta: every file under DIR, in
|
||||
subfolders too, whose suffix has a reader (.md, .markdown, .txt, and .pdf, .docx, .xlsx, .pptx through
|
||||
okfwiki.readers). A PDF or Office file is frozen as the text a reader of it sees (its safe name plus ".md"),
|
||||
with the file itself copied read-only to original/ (`original_copy`), `reader` (the type), `original_sha256`,
|
||||
`places` (the page, paragraph, cell or slide of each line, null
|
||||
for a blank line or a heading) and `left_out` (hidden parts left out, also reported); every document records
|
||||
`lang` (common.language) when its words tell it, for the view; one that cannot be read
|
||||
or has no text is skipped and named, and a PDF without pypdf at the pinned version is refused with exit 1
|
||||
and the install command. Each gets a safe name under kilde/: its path
|
||||
relative to DIR with "/" as "__", letters spelled in ASCII (æ ø å as ae oe aa, accents dropped), lower case,
|
||||
any other character as "-"; a name that would read as a release note (release-vX.Y.Z.md) ends in "-doc.md"
|
||||
instead, since the release-note rules go by the name. The entry keeps the path as it was (`original`, NFC), the file's own URL (`url`,
|
||||
file:///) and what was changed to make it a source (`normalised`, [] when the copy is byte for byte): "bom" (a
|
||||
UTF-8 byte order mark dropped), "cp1252" (decoded as Windows-1252, the Western Latin-1 superset, when not
|
||||
UTF-8) and "line-breaks" (\r\n, \r, form feed and every other line break written as \n), so a quote
|
||||
stands verbatim in the frozen copy. Kind "document", one main topic. A file is skipped, and named with the
|
||||
reason, when it is hidden (a name starting with "."; a hidden folder is skipped whole), has no reader, is
|
||||
empty or is a link. Two files with one safe name are refused with exit 1; exit 2 when no file is left, or DIR
|
||||
is not a folder, or ROOT exists. The manifest names the folder read (`folder`, its resolved path), so the copy
|
||||
can be read again from itself.
|
||||
subset A part of ROOT with the listed paths (one kilde/... per line, sorted, no duplicates), its own frozen_at, the
|
||||
index unchanged and subset_of pointing at ROOT's frozen_at and the sha256 of ROOT's manifest. Exit 2 when ROOT
|
||||
fails check, the paths file is not as described, or ROOT2 exists.
|
||||
diff Which sources of NEW are new, changed or gone since OLD, by path. Changed = another sha256 or another kind (the
|
||||
kind decides what a source is to the build). Exit 0 = no change, 1 = at least one, 2 = cannot run (a root
|
||||
fails check, or the two roots name another collection).
|
||||
marks The pages of COLLECTION (every .md under it, named by its path with "/") that cite a changed or gone source
|
||||
of the diff as sources[].resource "/" + path. New sources mark nothing. A page without frontmatter cites
|
||||
nothing; one whose frontmatter cannot be read stops marks. Exit 0 = no page marked, 1 = at least one, 2 =
|
||||
cannot run.
|
||||
Both print their report with the one written form, so two runs are byte-identical. A source read from a folder
|
||||
is named in diff with its `original` path too, as the user knows the file.
|
||||
reread Reads the folder ROOT was read from again, as the same collection, into ROOT2 (the folder command's rules). The
|
||||
folder is the manifest's `folder`, or for a copy frozen before that field, the one folder its entries name
|
||||
(each url less its original path). Exit 2 when ROOT fails check, was not read from a folder, its entries name
|
||||
two folders, the folder is gone, or ROOT2 exists.
|
||||
watch The change warning for a folder, one command for any scheduler (motor B15): reads the folder of ROOT again into
|
||||
DIR/latest (replaced whole each run), writes diff.json there and, with --bundle, marks.json, and prints each
|
||||
difference by its original path with the pages marked on it. DIR/watch-state.json keeps the lines reported,
|
||||
so a run says only what is new since the last one (a new frozen copy starts over). Exit 0 = nothing new,
|
||||
1 = something new, 2 = cannot run (also DIR inside the watched folder: the next reading would take the
|
||||
watch's files for documents). Only a warning: the pages are rebuilt from DIR/latest in a session.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import urllib.parse
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from okfwiki import readers
|
||||
from okfwiki.common import RunError, ascii_fold, language
|
||||
from okfwiki.kblocks import RELEASE
|
||||
|
||||
SCHEMA = "okf-wiki/source-manifest/1"
|
||||
META_SCHEMA = "okf-wiki/source-meta/1"
|
||||
DIFF_SCHEMA = "okf-wiki/source-diff/1"
|
||||
MARKS_SCHEMA = "okf-wiki/source-marks/1"
|
||||
MANIFEST = "source-manifest.json"
|
||||
KILDE = "kilde"
|
||||
PATH = re.compile(r"^kilde/[a-z0-9._-]+$")
|
||||
NAME = re.compile(r"^[a-z0-9._-]+$")
|
||||
HEX64 = re.compile(r"^[0-9a-f]{64}$")
|
||||
FROZEN_AT = re.compile(r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$")
|
||||
READ_ONLY = 0o444
|
||||
|
||||
|
||||
class Refused(Exception):
|
||||
"""The input breaks a rule; nothing was written. A command reports the problems with exit 1."""
|
||||
|
||||
def __init__(self, problems: list[str]) -> None:
|
||||
super().__init__("; ".join(problems))
|
||||
self.problems = problems
|
||||
|
||||
|
||||
def dump(data: Any) -> str:
|
||||
"""The one written form of every file here, so two writings of the same content are byte-identical."""
|
||||
return json.dumps(data, indent=1, sort_keys=True, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def sha256(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def ambiguous_lines(text: str) -> bool:
|
||||
"""True when splitlines() and split("\\n") (less a final empty part) give different lines."""
|
||||
parts = text.split("\n")
|
||||
if parts[-1] == "":
|
||||
parts.pop()
|
||||
return parts != text.splitlines()
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def load_json(path: Path) -> Any:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError:
|
||||
raise RunError(f"{path} does not exist") from None
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
raise RunError(f"{path} is not JSON: {exc}") from None
|
||||
|
||||
|
||||
def load_manifest(root: Path) -> dict[str, Any]:
|
||||
"""The manifest of ROOT, or RunError when there is none that check can read."""
|
||||
if not root.is_dir():
|
||||
raise RunError(f"{root} is not a directory")
|
||||
doc = load_json(root / MANIFEST)
|
||||
if not isinstance(doc, dict):
|
||||
raise RunError(f"{root / MANIFEST} is not a JSON object")
|
||||
if doc.get("schema") != SCHEMA:
|
||||
raise RunError(f"{root / MANIFEST} has schema {doc.get('schema')!r}, not {SCHEMA}")
|
||||
if not isinstance(doc.get("sources"), list):
|
||||
raise RunError(f"{root / MANIFEST} has no list of sources")
|
||||
return doc
|
||||
|
||||
|
||||
def nonempty_str(value: Any) -> bool:
|
||||
return isinstance(value, str) and value != ""
|
||||
|
||||
|
||||
def describe_problems(meta: Any) -> list[str]:
|
||||
"""The rules for what the meta says about one source (K1 rules 3, 4, 6)."""
|
||||
if not isinstance(meta, dict):
|
||||
return ["is not an object"]
|
||||
out = []
|
||||
if not nonempty_str(meta.get("kind")):
|
||||
out.append("kind is not a non-empty string")
|
||||
if not isinstance(meta.get("main_topic"), bool):
|
||||
out.append(f"main_topic is {meta.get('main_topic')!r}, not true or false")
|
||||
if not nonempty_str(meta.get("url")):
|
||||
out.append("url is not a non-empty string")
|
||||
if meta.get("kind") == "release-notes" and not nonempty_str(meta.get("version")):
|
||||
out.append("release-notes has no version")
|
||||
if "version" in meta and not nonempty_str(meta["version"]):
|
||||
out.append("version is not a non-empty string")
|
||||
return out
|
||||
|
||||
|
||||
def index_problem(value: Any) -> str | None:
|
||||
if (isinstance(value, dict) and isinstance(value.get("pages"), int) and not isinstance(value["pages"], bool)
|
||||
and isinstance(value.get("sha256"), str) and HEX64.match(value["sha256"]) and nonempty_str(value.get("url"))):
|
||||
return None
|
||||
return f"index is {value!r}, not {{pages, sha256, url}}"
|
||||
|
||||
|
||||
def subset_of_problem(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if (isinstance(value, dict) and isinstance(value.get("frozen_at"), str) and FROZEN_AT.match(value["frozen_at"])
|
||||
and isinstance(value.get("manifest_sha256"), str) and HEX64.match(value["manifest_sha256"])):
|
||||
return None
|
||||
return f"subset_of is {value!r}, not null or {{frozen_at, manifest_sha256}}"
|
||||
|
||||
|
||||
def file_facts(data: bytes) -> tuple[dict[str, Any], str | None]:
|
||||
"""sha256, bytes and lines of a source file, and the reason it cannot be a source, if any."""
|
||||
facts: dict[str, Any] = {"bytes": len(data), "sha256": sha256(data)}
|
||||
try:
|
||||
text = data.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return facts, "is not UTF-8"
|
||||
facts["lines"] = len(text.splitlines())
|
||||
if ambiguous_lines(text):
|
||||
return facts, "has ambiguous lines (a line break other than \\n)"
|
||||
return facts, None
|
||||
|
||||
|
||||
def check(root: Path) -> list[str]:
|
||||
"""K1 rule 11 and the form of the manifest; [] when the root is sound. RunError when check cannot run."""
|
||||
doc = load_manifest(root)
|
||||
problems: list[str] = []
|
||||
if not nonempty_str(doc.get("collection")):
|
||||
problems.append("collection is not a non-empty string")
|
||||
if not (isinstance(doc.get("frozen_at"), str) and FROZEN_AT.match(doc["frozen_at"])):
|
||||
problems.append(f"frozen_at is {doc.get('frozen_at')!r}, not YYYY-MM-DDTHH:MM:SSZ in UTC")
|
||||
if "index" in doc and (p := index_problem(doc["index"])):
|
||||
problems.append(p)
|
||||
if "folder" in doc and not (nonempty_str(doc["folder"]) and Path(doc["folder"]).is_absolute()):
|
||||
problems.append(f"folder is {doc['folder']!r}, not the absolute path of the folder the copy was read from")
|
||||
if "subset_of" not in doc:
|
||||
problems.append("subset_of is missing (null for a full copy)")
|
||||
elif p := subset_of_problem(doc["subset_of"]):
|
||||
problems.append(p)
|
||||
listed: set[str] = set()
|
||||
paths: list[str] = []
|
||||
for n, e in enumerate(doc["sources"], 1):
|
||||
if not isinstance(e, dict) or not isinstance(e.get("path"), str):
|
||||
problems.append(f"sources entry {n} has no path")
|
||||
continue
|
||||
path = e["path"]
|
||||
paths.append(path)
|
||||
if not PATH.match(path):
|
||||
problems.append(f"{path}: path is not kilde/ followed by [a-z0-9._-]")
|
||||
continue
|
||||
if path in listed:
|
||||
problems.append(f"{path}: listed more than once")
|
||||
continue
|
||||
listed.add(path)
|
||||
problems += [f"{path}: {p}" for p in describe_problems(e)]
|
||||
f = root / path
|
||||
if not f.is_file() or f.is_symlink():
|
||||
problems.append(f"{path}: missing")
|
||||
continue
|
||||
facts, why = file_facts(f.read_bytes())
|
||||
for key in ("sha256", "bytes", "lines"):
|
||||
if key in facts and e.get(key) != facts[key]:
|
||||
problems.append(f"{path}: {key} is {facts[key]!r} on disk, {e.get(key)!r} in the manifest")
|
||||
if why:
|
||||
problems.append(f"{path}: {why}")
|
||||
if "places" in e and not (isinstance(e["places"], list) and len(e["places"]) == facts.get("lines")):
|
||||
problems.append(f"{path}: places do not name one place per line ({facts.get('lines')} lines)")
|
||||
if "original_copy" in e:
|
||||
copy = e["original_copy"]
|
||||
c = root / copy if isinstance(copy, str) and re.fullmatch(rf"{ORIGINAL}/[a-z0-9._-]+", copy) else None
|
||||
if c is None:
|
||||
problems.append(f"{path}: original_copy {copy!r} is not {ORIGINAL}/ followed by [a-z0-9._-]")
|
||||
elif not c.is_file() or c.is_symlink():
|
||||
problems.append(f"{copy}: missing")
|
||||
elif sha256(c.read_bytes()) != e.get("original_sha256"):
|
||||
problems.append(f"{copy}: sha256 differs from original_sha256")
|
||||
if paths != sorted(paths):
|
||||
problems.append("sources are not sorted on path")
|
||||
kilde = root / KILDE
|
||||
if kilde.is_dir():
|
||||
for f in sorted(kilde.rglob("*")):
|
||||
rel = f.relative_to(root).as_posix()
|
||||
if (f.is_file() or f.is_symlink()) and rel not in listed:
|
||||
problems.append(f"{rel}: under kilde/ but not listed")
|
||||
return problems
|
||||
|
||||
|
||||
def write_root(out: Path, files: dict[str, bytes], doc: dict[str, Any], others: dict[str, bytes] | None = None) -> None:
|
||||
"""Write a root in a temporary folder beside OUT and move it into place, so a failure leaves no half root. files
|
||||
go under kilde/; others (relative path -> bytes) beside it, such as the original copies under original/."""
|
||||
if out.exists():
|
||||
raise RunError(f"{out} exists; refusing to overwrite")
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = Path(tempfile.mkdtemp(prefix=f".{out.name}-", dir=out.parent))
|
||||
try:
|
||||
(tmp / KILDE).mkdir()
|
||||
for name, data in files.items():
|
||||
p = tmp / KILDE / name
|
||||
p.write_bytes(data)
|
||||
os.chmod(p, READ_ONLY)
|
||||
for rel, data in (others or {}).items():
|
||||
p = tmp / rel
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_bytes(data)
|
||||
os.chmod(p, READ_ONLY)
|
||||
(tmp / MANIFEST).write_text(dump(doc), encoding="utf-8")
|
||||
problems = check(tmp)
|
||||
if problems:
|
||||
raise Refused(problems)
|
||||
tmp.rename(out)
|
||||
except BaseException:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
raise
|
||||
|
||||
|
||||
def checked_frozen_at(value: str | None) -> str:
|
||||
frozen_at = value or utc_now()
|
||||
if not FROZEN_AT.match(frozen_at):
|
||||
raise RunError(f"--frozen-at {frozen_at!r} is not YYYY-MM-DDTHH:MM:SSZ")
|
||||
return frozen_at
|
||||
|
||||
|
||||
def freeze(staging: Path, meta_path: Path, out: Path, frozen_at: str | None = None) -> dict[str, Any]:
|
||||
when = checked_frozen_at(frozen_at)
|
||||
if out.exists():
|
||||
raise RunError(f"{out} exists; refusing to overwrite")
|
||||
if not staging.is_dir():
|
||||
raise RunError(f"{staging} is not a directory")
|
||||
meta = load_json(meta_path)
|
||||
if not isinstance(meta, dict) or meta.get("schema") != META_SCHEMA or not isinstance(meta.get("sources"), dict):
|
||||
raise RunError(f"{meta_path} is not a {META_SCHEMA} file with an object of sources")
|
||||
described: dict[str, Any] = meta["sources"]
|
||||
problems: list[str] = []
|
||||
if not nonempty_str(meta.get("collection")):
|
||||
problems.append("meta: collection is not a non-empty string")
|
||||
if "index" in meta and (p := index_problem(meta["index"])):
|
||||
problems.append(f"meta: {p}")
|
||||
on_disk = {p.name: p for p in staging.iterdir()}
|
||||
for name in sorted(set(on_disk) - set(described)):
|
||||
problems.append(f"{name}: in the staging folder but not in the meta")
|
||||
for name in sorted(set(described) - set(on_disk)):
|
||||
problems.append(f"{name}: in the meta but not in the staging folder")
|
||||
files: dict[str, bytes] = {}
|
||||
entries = []
|
||||
for name in sorted(set(on_disk) & set(described)):
|
||||
if not NAME.match(name) or not on_disk[name].is_file() or on_disk[name].is_symlink():
|
||||
problems.append(f"{name}: not a plain file named [a-z0-9._-]")
|
||||
continue
|
||||
problems += [f"{name}: {p}" for p in describe_problems(described[name])]
|
||||
data = on_disk[name].read_bytes()
|
||||
facts, why = file_facts(data)
|
||||
if why:
|
||||
problems.append(f"{name}: {why}")
|
||||
continue
|
||||
files[name] = data
|
||||
entries.append({**described[name], **facts, "path": f"{KILDE}/{name}"})
|
||||
if problems:
|
||||
raise Refused(problems)
|
||||
doc: dict[str, Any] = {"collection": meta["collection"], "frozen_at": when, "schema": SCHEMA,
|
||||
"sources": entries, "subset_of": None}
|
||||
if "index" in meta:
|
||||
doc["index"] = meta["index"]
|
||||
write_root(out, files, doc)
|
||||
return doc
|
||||
|
||||
|
||||
TEXT_READERS = (".md", ".markdown", ".txt")
|
||||
READERS = TEXT_READERS + readers.TYPES
|
||||
# Every line break str.splitlines() knows besides \n; a source keeps \n only, so line N is the same for every reader.
|
||||
LINE_BREAKS = re.compile("\r\n|[\r\x0b\x0c\x1c\x1d\x1e\x85\u2028\u2029]")
|
||||
FOLDER_KIND = "document"
|
||||
ORIGINAL = "original" # beside kilde/: the PDF and Office files as they were, so the view can open them at the place
|
||||
|
||||
|
||||
def safe_name(rel: str) -> str:
|
||||
"""The kilde/ name of a file at rel (a path with "/") in a folder: see the folder command."""
|
||||
parts = [re.sub(r"-+", "-", re.sub(r"[^a-z0-9._-]", "-", ascii_fold(part))).strip("-") or "x"
|
||||
for part in rel.split("/")]
|
||||
return "__".join(parts)
|
||||
|
||||
|
||||
def normalised_text(data: bytes) -> tuple[bytes, list[str]]:
|
||||
"""The bytes of a text file as a source (UTF-8, lines broken at \n only) and what was changed to get them."""
|
||||
changed: list[str] = []
|
||||
if data.startswith(b"\xef\xbb\xbf"):
|
||||
data, changed = data[3:], ["bom"]
|
||||
try:
|
||||
text = data.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
try:
|
||||
text = data.decode("cp1252")
|
||||
except UnicodeDecodeError:
|
||||
text = data.decode("latin-1")
|
||||
changed.append("cp1252")
|
||||
flat = LINE_BREAKS.sub("\n", text)
|
||||
if flat != text:
|
||||
changed.append("line-breaks")
|
||||
return flat.encode("utf-8"), changed
|
||||
|
||||
|
||||
def walk(folder: Path) -> tuple[list[tuple[str, Path]], list[str]]:
|
||||
"""(every file under folder that has a reader, as (relative path in NFC, path), sorted on the relative path), and
|
||||
one line per thing skipped with its reason."""
|
||||
found: list[tuple[str, Path]] = []
|
||||
skipped: list[str] = []
|
||||
|
||||
def visit(d: Path) -> None:
|
||||
for p in sorted(d.iterdir()):
|
||||
rel = unicodedata.normalize("NFC", p.relative_to(folder).as_posix())
|
||||
if p.name.startswith("."):
|
||||
skipped.append(f"{rel}: skipped, hidden")
|
||||
elif p.is_symlink():
|
||||
skipped.append(f"{rel}: skipped, a link")
|
||||
elif p.is_dir():
|
||||
visit(p)
|
||||
elif not p.is_file():
|
||||
skipped.append(f"{rel}: skipped, not a plain file")
|
||||
elif p.suffix.lower() not in READERS:
|
||||
skipped.append(f"{rel}: skipped, no reader for {p.suffix.lower() or 'a file without a suffix'}")
|
||||
else:
|
||||
found.append((rel, p))
|
||||
|
||||
visit(folder)
|
||||
return sorted(found), sorted(skipped)
|
||||
|
||||
|
||||
def freeze_folder(folder: Path, collection: str, out: Path,
|
||||
frozen_at: str | None = None) -> tuple[dict[str, Any], list[str]]:
|
||||
"""The folder command: (the manifest written, the lines of what was skipped)."""
|
||||
when = checked_frozen_at(frozen_at)
|
||||
if out.exists():
|
||||
raise RunError(f"{out} exists; refusing to overwrite")
|
||||
if not folder.is_dir():
|
||||
raise RunError(f"{folder} is not a directory")
|
||||
if not nonempty_str(collection):
|
||||
raise RunError("the collection has no name")
|
||||
found, skipped = walk(folder)
|
||||
files: dict[str, bytes] = {}
|
||||
entries: list[dict[str, Any]] = []
|
||||
origin: dict[str, list[str]] = {}
|
||||
missing: list[str] = []
|
||||
originals: dict[str, bytes] = {}
|
||||
for rel, path in found:
|
||||
raw = path.read_bytes()
|
||||
read: dict[str, Any] = {}
|
||||
if path.suffix.lower() in readers.TYPES:
|
||||
try:
|
||||
rendering = readers.render(path)
|
||||
except readers.NoReader as e:
|
||||
missing.append(str(e))
|
||||
continue
|
||||
except readers.Unreadable as e:
|
||||
skipped.append(f"{rel}: skipped, unreadable: {e}")
|
||||
continue
|
||||
if not rendering.text.strip():
|
||||
skipped.append(f"{rel}: skipped, no text (a scanned page needs text recognition first)"
|
||||
if path.suffix.lower() == ".pdf" else f"{rel}: skipped, no text")
|
||||
continue
|
||||
if rendering.left_out:
|
||||
skipped.append(f"{rel}: {rendering.left_out} hidden parts left out")
|
||||
data, name = rendering.text.encode("utf-8"), safe_name(rel) + ".md"
|
||||
changed: list[str] = []
|
||||
copy = f"{ORIGINAL}/{safe_name(rel)}"
|
||||
originals[copy] = raw
|
||||
read = {"left_out": rendering.left_out, "original_copy": copy, "original_sha256": sha256(raw),
|
||||
"places": rendering.locators, "reader": path.suffix.lower()[1:]}
|
||||
else:
|
||||
data, changed = normalised_text(raw)
|
||||
if not data.strip():
|
||||
skipped.append(f"{rel}: skipped, empty")
|
||||
continue
|
||||
name = safe_name(rel)
|
||||
if RELEASE.match(name): # a folder's document has one main topic, never a release note's name (B13)
|
||||
name = name[:-len(".md")] + "-doc.md"
|
||||
origin.setdefault(name, []).append(rel)
|
||||
files[name] = data
|
||||
facts, _ = file_facts(data)
|
||||
lang = language(data.decode("utf-8"))
|
||||
entries.append({"kind": FOLDER_KIND, "main_topic": True, "normalised": changed, "original": rel,
|
||||
"url": path.resolve().as_uri(), **facts, **read, **({"lang": lang} if lang else {}),
|
||||
"path": f"{KILDE}/{name}"})
|
||||
problems = sorted(set(missing)) + [f"{name}: the safe name of {' and '.join(rels)}"
|
||||
for name, rels in sorted(origin.items()) if len(rels) > 1]
|
||||
if problems:
|
||||
raise Refused(problems)
|
||||
if not entries:
|
||||
raise RunError(f"{folder} holds no document with a reader ({', '.join(READERS)})")
|
||||
entries.sort(key=lambda e: str(e["path"]))
|
||||
doc: dict[str, Any] = {"collection": collection, "folder": str(folder.resolve()), "frozen_at": when,
|
||||
"schema": SCHEMA, "sources": entries, "subset_of": None}
|
||||
write_root(out, files, doc, originals)
|
||||
return doc, sorted(skipped)
|
||||
|
||||
|
||||
def subset(root: Path, paths_file: Path, out: Path, frozen_at: str | None = None) -> dict[str, Any]:
|
||||
when = checked_frozen_at(frozen_at)
|
||||
if out.exists():
|
||||
raise RunError(f"{out} exists; refusing to overwrite")
|
||||
problems = check(root)
|
||||
if problems:
|
||||
raise RunError(f"{root} fails check: {'; '.join(problems)}")
|
||||
text = paths_file.read_text(encoding="utf-8")
|
||||
wanted = text.split("\n")
|
||||
if wanted[-1] == "":
|
||||
wanted.pop()
|
||||
if not wanted:
|
||||
raise RunError(f"{paths_file} lists no path")
|
||||
if "" in wanted or wanted != sorted(set(wanted)):
|
||||
raise RunError(f"{paths_file} is not one path per line, sorted, without duplicates or empty lines")
|
||||
manifest_bytes = (root / MANIFEST).read_bytes()
|
||||
doc: dict[str, Any] = json.loads(manifest_bytes)
|
||||
by_path = {e["path"]: e for e in doc["sources"]}
|
||||
unknown = [p for p in wanted if p not in by_path]
|
||||
if unknown:
|
||||
raise RunError(f"not in {root / MANIFEST}: {', '.join(unknown)}")
|
||||
part = {**doc, "frozen_at": when, "sources": [by_path[p] for p in wanted],
|
||||
"subset_of": {"frozen_at": doc["frozen_at"], "manifest_sha256": sha256(manifest_bytes)}}
|
||||
write_root(out, {p[len(KILDE) + 1:]: (root / p).read_bytes() for p in wanted}, part,
|
||||
{e["original_copy"]: (root / e["original_copy"]).read_bytes() for e in part["sources"]
|
||||
if "original_copy" in e})
|
||||
return part
|
||||
|
||||
|
||||
def sound_manifest(root: Path) -> tuple[dict[str, Any], dict[str, str]]:
|
||||
"""The manifest of a root that passes check, and its stamp (frozen_at and the sha256 of the manifest's bytes)."""
|
||||
problems = check(root)
|
||||
if problems:
|
||||
raise RunError(f"{root} fails check: {'; '.join(problems)}")
|
||||
data = (root / MANIFEST).read_bytes()
|
||||
doc: dict[str, Any] = json.loads(data)
|
||||
return doc, {"frozen_at": doc["frozen_at"], "manifest_sha256": sha256(data)}
|
||||
|
||||
|
||||
def diff(old: Path, new: Path) -> dict[str, Any]:
|
||||
"""K4 item 1: the sources of NEW that are new, changed or gone since OLD, each list sorted on path."""
|
||||
old_doc, old_stamp = sound_manifest(old)
|
||||
new_doc, new_stamp = sound_manifest(new)
|
||||
if old_doc["collection"] != new_doc["collection"]:
|
||||
raise RunError(f"{old} is collection {old_doc['collection']!r} and {new} is {new_doc['collection']!r}; "
|
||||
"the paths of two collections can only match by chance")
|
||||
before = {e["path"]: e for e in old_doc["sources"]}
|
||||
after = {e["path"]: e for e in new_doc["sources"]}
|
||||
changed: list[dict[str, Any]] = []
|
||||
gone: list[dict[str, Any]] = []
|
||||
new_sources: list[dict[str, Any]] = []
|
||||
unchanged = 0
|
||||
for path in sorted(before.keys() | after.keys()):
|
||||
a, b = before.get(path), after.get(path)
|
||||
if b is None:
|
||||
assert a is not None
|
||||
gone.append({"kind": a["kind"], **as_known(a), "path": path})
|
||||
elif a is None:
|
||||
new_sources.append({"kind": b["kind"], **as_known(b), "path": path})
|
||||
elif a["sha256"] != b["sha256"] or a["kind"] != b["kind"]:
|
||||
changed.append({"kind": b["kind"], "new_sha256": b["sha256"], "old_sha256": a["sha256"], **as_known(b),
|
||||
"path": path})
|
||||
else:
|
||||
unchanged += 1
|
||||
return {"changed": changed, "gone": gone, "new": new_stamp, "new_sources": new_sources, "old": old_stamp,
|
||||
"schema": DIFF_SCHEMA, "unchanged": unchanged}
|
||||
|
||||
|
||||
def as_known(entry: dict[str, Any]) -> dict[str, Any]:
|
||||
"""The file as the user knows it ({"original": path in the folder}) for a source read from a folder, else {}."""
|
||||
return {"original": entry["original"]} if isinstance(entry.get("original"), str) else {}
|
||||
|
||||
|
||||
def has_change(report: dict[str, Any]) -> bool:
|
||||
return bool(report["changed"] or report["gone"] or report["new_sources"])
|
||||
|
||||
|
||||
def read_diff(path: Path) -> dict[str, str]:
|
||||
"""The changed and gone paths of a diff file, by path: "changed" or "gone". RunError when it is not a K4 diff."""
|
||||
doc = load_json(path)
|
||||
if not isinstance(doc, dict) or doc.get("schema") != DIFF_SCHEMA:
|
||||
raise RunError(f"{path} is not a {DIFF_SCHEMA} file")
|
||||
out: dict[str, str] = {}
|
||||
for change in ("changed", "gone"):
|
||||
entries = doc.get(change)
|
||||
if not isinstance(entries, list) or not all(isinstance(e, dict) and isinstance(e.get("path"), str)
|
||||
for e in entries):
|
||||
raise RunError(f"{path}: {change} is not a list of entries with a path")
|
||||
out.update({e["path"]: change for e in entries})
|
||||
return out
|
||||
|
||||
|
||||
def page_resources(path: Path, rel: str) -> list[str]:
|
||||
"""Every sources[].resource string in the frontmatter of one page; [] for a page without frontmatter."""
|
||||
lines = path.read_text(encoding="utf-8").split("\n")
|
||||
if lines[0].strip() != "---":
|
||||
return []
|
||||
end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
|
||||
if end is None:
|
||||
raise RunError(f"{rel}: the frontmatter is not closed with ---")
|
||||
try:
|
||||
data = yaml.safe_load("\n".join(lines[1:end]))
|
||||
except yaml.YAMLError as exc:
|
||||
raise RunError(f"{rel}: the frontmatter does not parse: {str(exc).splitlines()[0]}") from None
|
||||
if data is None:
|
||||
return []
|
||||
if not isinstance(data, dict):
|
||||
raise RunError(f"{rel}: the frontmatter is not a mapping")
|
||||
cited = data.get("sources") or []
|
||||
if not isinstance(cited, list):
|
||||
raise RunError(f"{rel}: sources is not a list")
|
||||
return [s["resource"] for s in cited if isinstance(s, dict) and isinstance(s.get("resource"), str)]
|
||||
|
||||
|
||||
def marks(collection: Path, diff_path: Path) -> dict[str, Any]:
|
||||
"""K4 item 2: the pages built on a changed or gone source, sorted on page, their sources sorted on path."""
|
||||
changes = read_diff(diff_path)
|
||||
if not collection.is_dir():
|
||||
raise RunError(f"{collection} is not a directory")
|
||||
pages: list[dict[str, Any]] = []
|
||||
for p in sorted(collection.rglob("*.md")):
|
||||
if not p.is_file():
|
||||
continue
|
||||
rel = p.relative_to(collection).as_posix()
|
||||
hit = {r[1:] for r in page_resources(p, rel) if r.startswith("/") and r[1:] in changes}
|
||||
if hit:
|
||||
pages.append({"page": rel, "sources": [{"change": changes[s], "path": s} for s in sorted(hit)]})
|
||||
pages.sort(key=lambda e: e["page"])
|
||||
return {"pages": pages, "schema": MARKS_SCHEMA}
|
||||
|
||||
|
||||
WATCH_SCHEMA = "okf-wiki/folder-watch/1"
|
||||
WATCH_STATE = "watch-state.json"
|
||||
LATEST = "latest"
|
||||
|
||||
|
||||
def source_folder(doc: dict[str, Any]) -> Path:
|
||||
"""The folder a copy was read from: its `folder`, or for a copy frozen before that field, the one folder its
|
||||
entries name (each `url` less its `original` path). RunError when the copy was not read from a folder."""
|
||||
if nonempty_str(doc.get("folder")):
|
||||
return Path(doc["folder"])
|
||||
found: set[Path] = set()
|
||||
for e in doc["sources"]:
|
||||
url, original = e.get("url"), e.get("original")
|
||||
if not (isinstance(url, str) and url.startswith("file://") and nonempty_str(original)):
|
||||
continue
|
||||
path = Path(urllib.parse.unquote(urllib.parse.urlparse(url).path))
|
||||
depth = len(Path(original).parts)
|
||||
if len(path.parts) <= depth:
|
||||
continue
|
||||
found.add(Path(*path.parts[:-depth]))
|
||||
if not found:
|
||||
raise RunError("the copy was not read from a folder (no folder, and no entry with a file:// url and an "
|
||||
"original path); only a folder copy can be read again")
|
||||
if len(found) > 1:
|
||||
raise RunError(f"the entries name more than one folder ({', '.join(sorted(map(str, found)))}); "
|
||||
"read the folder again with the folder command")
|
||||
return found.pop()
|
||||
|
||||
|
||||
def sound_root(root: Path) -> dict[str, Any]:
|
||||
problems = check(root)
|
||||
if problems:
|
||||
raise RunError(f"{root} fails check: {'; '.join(problems[:5])}")
|
||||
return load_manifest(root)
|
||||
|
||||
|
||||
def reread(root: Path, out: Path, frozen_at: str | None = None) -> tuple[dict[str, Any], list[str], Path]:
|
||||
"""The reread command: the folder ROOT was read from, read again as the same collection into OUT."""
|
||||
doc = sound_root(root)
|
||||
folder = source_folder(doc)
|
||||
if not folder.is_dir():
|
||||
raise RunError(f"the folder {folder} that {root} was read from is not there")
|
||||
try:
|
||||
new, skipped = freeze_folder(folder, doc["collection"], out, frozen_at)
|
||||
except Refused as exc:
|
||||
raise RunError(f"{folder} cannot be read again: {'; '.join(exc.problems)}") from exc
|
||||
return new, skipped, folder
|
||||
|
||||
|
||||
def watch_lines(report: dict[str, Any]) -> list[str]:
|
||||
"""One line per difference, "change path [new sha256]", so a later change of the same file is a new line."""
|
||||
lines = [f"new {e['path']}" for e in report["new_sources"]]
|
||||
lines += [f"changed {e['path']} {e['new_sha256']}" for e in report["changed"]]
|
||||
lines += [f"gone {e['path']}" for e in report["gone"]]
|
||||
return sorted(lines)
|
||||
|
||||
|
||||
def watch(root: Path, into: Path, bundle: Path | None, frozen_at: str | None = None) -> tuple[str, bool]:
|
||||
"""The watch command: (the report, whether something is new since the last run). See the module doc."""
|
||||
doc = sound_root(root)
|
||||
folder = source_folder(doc).resolve()
|
||||
target = into.resolve()
|
||||
if target == folder or folder in target.parents:
|
||||
raise RunError(f"{into} is inside the folder {folder}; the next reading would take the watch's files for "
|
||||
"documents. Put it beside the folder")
|
||||
state_path = into / WATCH_STATE
|
||||
state = load_json(state_path) if state_path.is_file() else None
|
||||
if state is not None and not (isinstance(state, dict) and state.get("schema") == WATCH_SCHEMA
|
||||
and isinstance(state.get("entries"), list)):
|
||||
raise RunError(f"{state_path} is not a watch state ({WATCH_SCHEMA})")
|
||||
into.mkdir(parents=True, exist_ok=True)
|
||||
work = Path(tempfile.mkdtemp(prefix=".watch-", dir=into))
|
||||
try:
|
||||
_, skipped, _ = reread(root, work / LATEST, frozen_at)
|
||||
report = diff(root, work / LATEST)
|
||||
(work / LATEST / "diff.json").write_text(dump(report), encoding="utf-8")
|
||||
marked: dict[str, int] = {}
|
||||
pages = 0
|
||||
if bundle is not None:
|
||||
found = marks(bundle, work / LATEST / "diff.json")
|
||||
(work / LATEST / "marks.json").write_text(dump(found), encoding="utf-8")
|
||||
pages = len(found["pages"])
|
||||
for pg in found["pages"]:
|
||||
for src in pg["sources"]:
|
||||
marked[src["path"]] = marked.get(src["path"], 0) + 1
|
||||
latest = into / LATEST
|
||||
if latest.exists():
|
||||
latest.rename(work / "replaced")
|
||||
(work / LATEST).rename(latest)
|
||||
finally:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
|
||||
entries = watch_lines(report)
|
||||
same_copy = state is not None and state.get("frozen") == report["old"]
|
||||
seen = {str(e) for e in state["entries"]} if state is not None and same_copy else set()
|
||||
news = [e for e in entries if e not in seen]
|
||||
resolved = sorted(seen - set(entries))
|
||||
state_path.write_text(dump({"entries": entries, "frozen": report["old"], "schema": WATCH_SCHEMA}),
|
||||
encoding="utf-8")
|
||||
by_path = {e["path"]: e for k in ("new_sources", "changed", "gone") for e in report[k]}
|
||||
counts = (f"{len(report['new_sources'])} new, {len(report['changed'])} changed, {len(report['gone'])} gone, "
|
||||
f"{report['unchanged']} unchanged")
|
||||
lines = [f"watch: {folder} read again into {latest}, against {root} (frozen {report['old']['frozen_at']}): "
|
||||
f"{counts}"]
|
||||
if not news:
|
||||
lines.append(" nothing differs" if not entries else " nothing new since the last run")
|
||||
else:
|
||||
lines.append("New since the last run:" if seen or same_copy else "Every difference (first run against this copy):")
|
||||
for e in news:
|
||||
change, path = e.split(" ")[:2]
|
||||
known = by_path[path].get("original", path)
|
||||
n = marked.get(path)
|
||||
lines.append(f" {change:<8} {known} ({path})" + (f", {n} page(s) marked" if n else ""))
|
||||
if resolved:
|
||||
lines.append("No longer differs from the frozen copy:")
|
||||
lines += [f" {e.split(' ')[1]}" for e in resolved]
|
||||
if skipped:
|
||||
lines.append("Skipped in the folder:")
|
||||
lines += [f" {line}" for line in skipped]
|
||||
if bundle is not None:
|
||||
lines.append(f"{pages} page(s) marked in {bundle}; the list: {latest / 'marks.json'}")
|
||||
lines.append(f"Every difference: {latest / 'diff.json'}. This only warns; rebuild from {latest} to update.")
|
||||
return "\n".join(lines) + "\n", bool(news)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
c = sub.add_parser("check")
|
||||
c.add_argument("root")
|
||||
f = sub.add_parser("freeze")
|
||||
f.add_argument("staging")
|
||||
f.add_argument("--meta", required=True)
|
||||
f.add_argument("--out", required=True)
|
||||
f.add_argument("--frozen-at")
|
||||
fo = sub.add_parser("folder")
|
||||
fo.add_argument("folder")
|
||||
fo.add_argument("--collection", required=True)
|
||||
fo.add_argument("--out", required=True)
|
||||
fo.add_argument("--frozen-at")
|
||||
s = sub.add_parser("subset")
|
||||
s.add_argument("root")
|
||||
s.add_argument("--paths", required=True)
|
||||
s.add_argument("--out", required=True)
|
||||
s.add_argument("--frozen-at")
|
||||
d = sub.add_parser("diff")
|
||||
d.add_argument("old")
|
||||
d.add_argument("new")
|
||||
m = sub.add_parser("marks")
|
||||
m.add_argument("collection")
|
||||
m.add_argument("--diff", required=True)
|
||||
r = sub.add_parser("reread")
|
||||
r.add_argument("root")
|
||||
r.add_argument("--out", required=True)
|
||||
r.add_argument("--frozen-at")
|
||||
w = sub.add_parser("watch")
|
||||
w.add_argument("root")
|
||||
w.add_argument("--into", required=True)
|
||||
w.add_argument("--bundle")
|
||||
w.add_argument("--frozen-at")
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
if args.command == "check":
|
||||
root = Path(args.root)
|
||||
problems = check(root)
|
||||
n = len(load_manifest(root)["sources"])
|
||||
if problems:
|
||||
print(f"check: {len(problems)} problems in {root} ({n} sources listed)")
|
||||
for p in problems:
|
||||
print(f" {p}")
|
||||
return 1
|
||||
print(f"check: {n} sources in {root}, every file present and listed")
|
||||
return 0
|
||||
if args.command == "diff":
|
||||
report = diff(Path(args.old), Path(args.new))
|
||||
sys.stdout.write(dump(report))
|
||||
return 1 if has_change(report) else 0
|
||||
if args.command == "marks":
|
||||
marked = marks(Path(args.collection), Path(args.diff))
|
||||
sys.stdout.write(dump(marked))
|
||||
return 1 if marked["pages"] else 0
|
||||
if args.command == "reread":
|
||||
new, skipped, folder = reread(Path(args.root), Path(args.out), args.frozen_at)
|
||||
print(f"reread: {len(new['sources'])} sources from {folder} -> {args.out}; {len(skipped)} skipped")
|
||||
for line in skipped:
|
||||
print(f" {line}")
|
||||
return 0
|
||||
if args.command == "watch":
|
||||
text, news = watch(Path(args.root), Path(args.into), Path(args.bundle) if args.bundle else None,
|
||||
args.frozen_at)
|
||||
sys.stdout.write(text)
|
||||
return 1 if news else 0
|
||||
if args.command == "folder":
|
||||
doc, skipped = freeze_folder(Path(args.folder), args.collection, Path(args.out), args.frozen_at)
|
||||
print(f"folder: {len(doc['sources'])} sources -> {args.out}; {len(skipped)} skipped")
|
||||
for line in skipped:
|
||||
print(f" {line}")
|
||||
return 0
|
||||
if args.command == "freeze":
|
||||
doc = freeze(Path(args.staging), Path(args.meta), Path(args.out), args.frozen_at)
|
||||
else:
|
||||
doc = subset(Path(args.root), Path(args.paths), Path(args.out), args.frozen_at)
|
||||
print(f"{args.command}: {len(doc['sources'])} sources -> {args.out}")
|
||||
return 0
|
||||
except Refused as exc:
|
||||
print(f"{args.command} refused: {len(exc.problems)} problems, nothing written", file=sys.stderr)
|
||||
for p in exc.problems:
|
||||
print(f" {p}", file=sys.stderr)
|
||||
return 1
|
||||
except (RunError, OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
print(f"{args.command} cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
31
okfwiki/transcript.py
Normal file
31
okfwiki/transcript.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"""Reads Claude Code session transcripts (.jsonl), as written under ~/.claude/projects/<slug>/. No model call."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
USAGE_FIELDS = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens", "output_tokens")
|
||||
|
||||
|
||||
def is_agent_type(recorded: object, name: str) -> bool:
|
||||
"""Whether a subagent's recorded agentType is the agent `name`, started bare ("okf-wiki-reader") or from a
|
||||
plugin, which Claude Code records with the plugin's namespace ("<plugin>:okf-wiki-reader")."""
|
||||
if not isinstance(recorded, str):
|
||||
return False
|
||||
plugin, sep, bare = recorded.rpartition(":")
|
||||
return bare == name and (not sep or bool(plugin))
|
||||
|
||||
|
||||
def read_transcript(path: Path) -> list[dict[str, Any]]:
|
||||
"""Every JSON object line of a transcript; a line that does not parse is skipped."""
|
||||
lines = []
|
||||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||||
try:
|
||||
line = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(line, dict):
|
||||
lines.append(line)
|
||||
return lines
|
||||
1092
okfwiki/view.py
Normal file
1092
okfwiki/view.py
Normal file
File diff suppressed because it is too large
Load diff
Loading…
Add table
Add a link
Reference in a new issue