"""The link pass: the model proposes links by meaning; this code strips, asks, accepts and places them. No model call. Usage: python3 -m okfwiki.klink strip BUNDLE --out DIR python3 -m okfwiki.klink sentences --accepted FILE --pack N --bundle DIR --out FILE python3 -m okfwiki.klink task BUNDLE --sentences FILE --catalogue IDS --out DIR python3 -m okfwiki.klink accept PROPOSALS --bundle DIR --links FILE python3 -m okfwiki.klink place BUNDLE --links FILE --out DIR [--replace] python3 -m okfwiki.klink pass-before --accepted FILE --pack N [--to M] --bundle DIR [--catalogue IDS] --out DIR python3 -m okfwiki.klink pass-after --pass-dir DIR --bundle DIR --links FILE --linked DIR --sources DIR --transcripts DIR --okter FILE --work DIR --kost FILE --lines L[,L] --cap N --unit U --done D --total T [--blocks FILE] [--name NAME] Links are data (`links.json`, schema okf-wiki/links/1), never edits to pages, because K re-assembles pages from fragments. A record is {page, sentence, anchor, at, target}: the sentence key, the anchor text and its character offset in that sentence, and the target page. The sentence key is the first 16 hex characters of the sha256 of the sentence's original text (sliced with audit.sentence_spans from the unmasked text) after footnote marks are removed, every markdown link is reduced to its anchor text and whitespace is collapsed, plus "-", the occurrence of that text on the page. Every command first reduces each bundle link in a concept body to its anchor text, in prose, code, headings and footnote definitions alike, so keys and offsets are computed on link-free text and placement is deterministic. index.md and log.md are copied unchanged. strip copies the bundle with every bundle link in a concept body reduced to its anchor text. sentences the sentence keys on assembled pages that come from pack N's accepted fragments (`pack`, `concept_id`, `text`). Exit 1 when the pack has no sentence (an empty denominator). task renders the task parts: the contract, the catalogue as `id | title` lines and, per page, only the listed sentences with their keys; parts are capped at 60,000 characters. A catalogue that alone exceeds the cap cannot run (exit 2): it has to be split or filtered first. accept validates proposals {"links": [{page, sentence, anchor, target}], "none": [ids]} and appends accepted records; rejects are counted per reason, `none` answers are counted, and no record is ever rewritten. place writes every record into a copy of the bundle; refuses an existing output unless --replace (then it writes a temporary directory and swaps it in). A record whose sentence no longer exists is stale: exit 1. pass-before / pass-after chain the non-model steps of one pass (sentences -> task; accept -> place -> link rule -> cost harvest -> cost stop) and print only counts, so a pass costs the orchestrator few turns. With --to M, pass-before covers packs N..M, one directory p per pack, and pass-after on the parent accepts every p in pack order: the same links as one pass per pack, for the same orchestrator turns. With --unit blocks, --done is the blocks of the accepted packs the pass covers (the done line of run-after for its last pack), and the stop line divides by the blocks of every pack with a compiler run: in the skill's chain the compiler of the next pack has run before this pass-after, so the harvest holds its run and both sides of the stop line cover the same packs. The orchestration is one line for the whole session and is not split per pack. A --done that is not those blocks, or a run on a capped line that is tied to no pack or to a pack the log lacks, cannot run (exit 2, STOP). The stop is measured cost plus the blocks of a compiled pack not yet accepted (cost stop --next, and --next-chars when the pack log has their characters), and the projection is reported only (the operator 12.10.2026). The catalogue of a pack is filtered to its candidate targets within CATALOGUE_CHARS (filter_catalogue) from a pool: the ids in --catalogue, or else the concepts with fragments in this or an earlier pack. Exit 0 done, 1 a counted failure (stale, empty, STOP), 2 cannot run. The guard (contract K2 point 17; kpack run-after calls it, it has no command of its own): `homeless` takes the pages with a no-main-page-link finding and the sources with a main-page-missing finding from the auditor's JSON, so the auditor is the only judge. `guard_parts` renders one task for the model with GUARD_CONTRACT, the pack's filtered catalogue of main pages, each page without a home (id, title, description) and each source without a main page with the pages that cite it; `read_guard` checks the answer: a home or none per page, a main page per source. The guard's task runs on the linker agent, because a home is a link from a main page chosen by meaning. """ from __future__ import annotations import argparse import hashlib import json import posixpath import re import shutil import sys import tempfile from collections import Counter from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any from okfwiki import audit, cost from okfwiki.common import RunError, write_new SCHEMA = "okf-wiki/links/1" SENTENCES_SCHEMA = "okf-wiki/link-sentences/1" PARTS_SCHEMA = "okf-wiki/link-parts/1" PART_CHARS = 60_000 CATALOGUE_CHARS = 32_000 # leaves ~27,000 characters of a part for sentences; order 28 A3: with the main pages first, the # fewest far concepts a pack of the rest saw in the model was 205 (24,000: three packs under G's 179) RECENT_PACKS = 2 MAX_LINKS = 3 PART_HEADER_ROOM = len("\n") FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})") REASONS = ("malformed", "sentence-missing", "target-missing", "self-link", "no-footnote", "anchor-not-in-sentence", "anchor-in-code", "duplicate", "over-limit", "overlap") CONTRACT = """## Contract You add links to sentences of a knowledge collection. Every sentence below has a key. Link by meaning, not by name: a link says that a phrase in the sentence is about a concept in the catalogue, and the phrase may use other words than the concept's title. - Targets: only ids from the catalogue in this task. Never link a page to itself. - Anchor: a phrase that already stands, character for character, in the sentence, outside backticks. Change no word. - At most 3 links in one sentence. No lists of related pages and no "see also" lines. - When nothing in the catalogue fits, put the page id in `none`. - Output: one JSON object `{"links": [{"page": ..., "sentence": ..., "anchor": ..., "target": ...}], "none": [...]}`, where `page` is the page file as the task names it, `sentence` is the sentence key and `target` is a catalogue id. Write it to the output path named in your prompt, then stop. """ GUARD_CONTRACT = """## Contract (home task) A main page is the page for the main topic of one docs source. Every other page needs a home: a main page that links to it in its list of related concepts. The pages and sources below lack one. Decide by meaning, as a reader of the main page would look for the page. - For each page under "Pages without a home": choose its home from the catalogue, or a page you name a main page below, and add {"page": , "home": } to `homes`. When no main page fits, put the page id in `none`: a page without a natural home is reported, never given a forced home. - For each source under "Sources without a main page": choose, from the pages listed under it, the page whose topic is that source's main topic, and add {"source": , "page": } to `main_pages`. - Output: one JSON object `{"homes": [...], "none": [...], "main_pages": [...]}`. Write it to the output path named in your prompt, then stop. """ GUARDED = frozenset({"no-main-page-link", "main-page-missing"}) GUARD_REASONS = ("malformed", "page-not-asked", "duplicate", "self-home", "home-not-in-catalogue", "source-not-asked", "page-not-a-candidate") KILDE_PATH = re.compile(r"(? tuple[str, str]: """(frontmatter block with its closing fence and newline, body); a page without frontmatter has an empty head.""" lines = text.split("\n") if not lines or lines[0].strip() != "---": return "", text end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None) if end is None: return "", text return "\n".join(lines[:end + 1]) + "\n", "\n".join(lines[end + 1:]) def reduce_links(body: str) -> tuple[str, int]: """Every bundle link reduced to its anchor text, wherever it stands in the body; external links are kept.""" count = 0 def one(m: re.Match[str]) -> str: nonlocal count if not audit.is_bundle_link(m.group(2)): return m.group(0) count += 1 return m.group(1) return audit.sub_links(one, body), count def is_concept(rel: str) -> bool: return rel.endswith(".md") and PurePosixPath(rel).name not in audit.RESERVED def key_text(text: str) -> str: plain = audit.sub_links(lambda m: m.group(1), audit.FOOTNOTE_REF.sub("", text)) return " ".join(plain.split()) def prose_spans(body: str) -> list[tuple[int, int]]: """(start, end) offsets of paragraphs and list items; headings, footnote definitions and fenced blocks are not prose.""" spans: list[tuple[int, int]] = [] current: list[int] | None = None fence = "" offset = 0 for raw in body.split("\n"): start, end = offset, offset + len(raw) offset = end + 1 f = FENCE.match(raw) if fence: if f and f.group(1)[0] == fence[0] and len(f.group(1)) >= len(fence): fence = "" continue if f: fence, current = f.group(1), None continue if not raw.strip() or audit.HEADING.match(raw) or audit.FOOTNOTE_DEF.match(raw.strip()): current = None continue item = audit.LIST_ITEM.match(raw) if item: current = [start + item.end(), end] spans.append((current[0], current[1])) elif current is None: current = [start + len(raw) - len(raw.lstrip()), end] spans.append((current[0], current[1])) else: current[1] = end spans[-1] = (current[0], current[1]) return spans def body_sentences(body: str) -> list[Sentence]: """The sentences of a link-reduced body, each with its key.""" out: list[Sentence] = [] seen: Counter[str] = Counter() for a, b in prose_spans(body): block = body[a:b] masked = audit.mask_code(block) for s, e in audit.sentence_spans(block): text = block[s:e] norm = key_text(text) digest = hashlib.sha256(norm.encode("utf-8")).hexdigest()[:16] labels = tuple(audit.FOOTNOTE_REF.findall(masked[s:e])) out.append(Sentence(f"{digest}-{seen[digest]}", text, masked[s:e], a + s, bool(labels), labels)) seen[digest] += 1 return out def page_sentences(text: str) -> list[Sentence]: return body_sentences(reduce_links(split_page(text)[1])[0]) @dataclass class Page: rel: str head: str body: str # link-reduced title: str resources: dict[str, str] # sources[].id -> resource description: str = "" def read_pages(bundle: Path) -> dict[str, Page]: pages: dict[str, Page] = {} for p in sorted(bundle.rglob("*.md")): rel = p.relative_to(bundle).as_posix() if not is_concept(rel): continue doc = audit.read_document(p, rel) head, body = split_page(p.read_text(encoding="utf-8")) fm = doc.frontmatter or {} srcs = fm.get("sources") resources = {str(s["id"]): str(s.get("resource", "")) for s in (srcs if isinstance(srcs, list) else []) if isinstance(s, dict) and "id" in s} pages[rel] = Page(rel, head, reduce_links(body)[0], str(fm.get("title") or PurePosixPath(rel).stem), resources, " ".join(str(fm.get("description") or "").split())) return pages def page_rel(name: str) -> str: name = name.strip() return name if name.endswith(".md") else name + ".md" def load_json(path: Path) -> Any: return json.loads(path.read_text(encoding="utf-8")) def dump(doc: Any) -> str: return json.dumps(doc, ensure_ascii=False, indent=1) + "\n" # --- strip and place ------------------------------------------------------------------------------------------- def copy_reduced(bundle: Path, out: Path, records: list[dict[str, Any]] | None) -> tuple[int, int, int]: """Copy bundle to out with every bundle link reduced, then place records; (links reduced, placed, stale).""" by_page: dict[str, list[dict[str, Any]]] = {} for r in records or []: by_page.setdefault(str(r["page"]), []).append(r) reduced = placed = stale = 0 for p in sorted(bundle.rglob("*")): if not p.is_file(): continue rel = p.relative_to(bundle).as_posix() dst = out / rel dst.parent.mkdir(parents=True, exist_ok=True) if not is_concept(rel): shutil.copyfile(p, dst) continue head, body = split_page(p.read_text(encoding="utf-8")) body, n = reduce_links(body) reduced += n body, ok, bad = place_records(rel, body, by_page.pop(rel, [])) placed, stale = placed + ok, stale + bad dst.write_text(head + body, encoding="utf-8") stale += sum(len(v) for v in by_page.values()) # records for pages that no longer exist return reduced, placed, stale def place_records(rel: str, body: str, records: list[dict[str, Any]]) -> tuple[str, int, int]: by_key = {s.key: s for s in body_sentences(body)} edits: list[tuple[int, int, str]] = [] stale = 0 for r in records: s = by_key.get(str(r["sentence"])) anchor, at = str(r["anchor"]), int(r["at"]) if s is None or s.text[at:at + len(anchor)] != anchor: stale += 1 continue target = posixpath.relpath(str(r["target"]), posixpath.dirname(rel) or ".") edits.append((s.start + at, s.start + at + len(anchor), f"[{anchor}]({target})")) for a, b, link in sorted(edits, reverse=True): body = body[:a] + link + body[b:] return body, len(edits), stale def strip(bundle: Path, out: Path) -> int: if out.exists(): raise RunError(f"{out} exists; refusing to overwrite") return copy_reduced(bundle, out, None)[0] def place(bundle: Path, links: Path, out: Path, replace: bool) -> tuple[int, int]: records = load_json(links)["links"] if out.exists() and not replace: raise RunError(f"{out} exists; refusing to overwrite without --replace") out.parent.mkdir(parents=True, exist_ok=True) tmp = Path(tempfile.mkdtemp(prefix=f".{out.name}-", dir=out.parent)) try: _, placed, stale = copy_reduced(bundle, tmp, records) if out.exists(): old = out.with_name(f".{out.name}-old") out.rename(old) tmp.rename(out) shutil.rmtree(old) else: tmp.rename(out) finally: if tmp.exists(): shutil.rmtree(tmp) return placed, stale # --- what the model is asked ----------------------------------------------------------------------------------- def pack_sentences(accepted: dict[str, Any], pack: int, pages: dict[str, Page]) -> dict[str, list[str]]: texts: dict[str, list[str]] = {} for f in accepted["fragments"]: if int(f["pack"]) == pack: texts.setdefault(page_rel(str(f["concept_id"])), []).append(key_text(str(f["text"]))) out: dict[str, list[str]] = {} for rel, frags in texts.items(): if rel not in pages: continue keys = [s.key for s in body_sentences(pages[rel].body) if s.footnoted and any(key_text(s.text).rstrip(".!?") in t for t in frags)] if keys: out[rel] = keys return out def concept_id(rel: str) -> str: return PurePosixPath(rel).with_suffix("").as_posix() def pack_pool(accepted: dict[str, Any], pack: int, pages: dict[str, Page]) -> list[str]: """The concepts with fragments in this or an earlier pack: the pages that exist after pack N in an additive build.""" ids = {concept_id(page_rel(str(f["concept_id"]))) for f in accepted["fragments"] if int(f["pack"]) <= pack} return sorted(cid for cid in ids if page_rel(cid) in pages) def filter_catalogue(accepted: dict[str, Any], pack: int, pool: list[str], pages: dict[str, Page], budget: int = CATALOGUE_CHARS) -> list[str]: """The candidate targets of a pack, ranked, cut at the first that does not fit the character budget. The main pages (accepted.json's main_pages) come first, then the rest, each in rank order (order 28 A3: they are the homes of criterion 6, and with the old rule a late pack of collection 1 saw few of them). Rank: (0) the concepts this pack writes to; (1) those last written to in the RECENT_PACKS packs before it, latest first, because the fixed order puts related sources next to each other; (2) the rest by how many distinct source files their page cites, most first, so a late pack can still reach the central concepts made early. Ties by id. The rule uses structure only (packs, citations and main pages), never names: the model links by meaning. """ last: dict[str, int] = {} for f in accepted["fragments"]: p, cid = int(f["pack"]), concept_id(page_rel(str(f["concept_id"]))) if p <= pack: last[cid] = max(last.get(cid, 0), p) def rank(cid: str) -> tuple[int, int, str]: lp = last.get(cid) if lp == pack: return 0, 0, cid if lp is not None and pack - lp <= RECENT_PACKS: return 1, -lp, cid return 2, -len(set(pages[page_rel(cid)].resources.values()) - {""}), cid mains = {concept_id(page_rel(str(m["concept_id"]))) for m in accepted.get("main_pages", [])} ranked = sorted(pool, key=rank) out: list[str] = [] used = 0 for cid in [c for c in ranked if c in mains] + [c for c in ranked if c not in mains]: size = len(catalogue_text([cid], pages)) if used + size > budget: break out.append(cid) used += size return out @dataclass class Section: page: str lines: list[str] heading: str def catalogue_text(ids: list[str], pages: dict[str, Page]) -> str: out = [] for cid in ids: rel = page_rel(cid) if rel not in pages: raise RunError(f"catalogue id {cid!r} is not a concept page in the bundle") out.append(f"{PurePosixPath(rel).with_suffix('').as_posix()} | {pages[rel].title}\n") return "".join(out) def part_text(label: str, catalogue: str, sections: list[Section]) -> str: body = "".join(f"### {s.heading}\n" + "".join(s.lines) + "\n" for s in sections) return (f"# Link pass: {label}\n\n" + CONTRACT + "\n## Catalogue (link targets)\n\n" + catalogue + "\n## Sentences\n\n" + body) def render_sections(label: str, fixed_ids: list[str], sections: list[Section], pages: dict[str, Page], part_chars: int) -> list[tuple[str, dict[str, Any]]]: fixed = catalogue_text(fixed_ids, pages) if len(fixed) > part_chars: raise RunError(f"the catalogue alone is {len(fixed)} characters ({len(fixed_ids)} concepts), over the part " f"limit of {part_chars}: split or filter the catalogue before this pass") room = part_chars - PART_HEADER_ROOM def size(group: list[Section]) -> int: return len(part_text(label, fixed, group)) if size([]) > room: raise RunError(f"the contract and catalogue leave no room for a sentence within {part_chars} characters") chunks: list[Section] = [] for s in sections: piece = Section(s.page, [], s.heading) for line in s.lines: if piece.lines and size([Section(s.page, piece.lines + [line], piece.heading)]) > room: chunks.append(piece) piece = Section(s.page, [], s.heading + " (continued)") piece.lines.append(line) if size([piece]) > room: raise RunError(f"one sentence of {s.page} does not fit in a part of {part_chars} characters") chunks.append(piece) groups: list[list[Section]] = [] for c in chunks: if groups and size(groups[-1] + [c]) <= room: groups[-1].append(c) else: groups.append([c]) out: list[tuple[str, dict[str, Any]]] = [] for i, group in enumerate(groups, 1): text = f"\n" + part_text(label, fixed, group) out.append((text, {"part": i, "file": f"part-{i}.md", "pages": len({s.page for s in group}), "sentences": sum(len(s.lines) for s in group), "catalogue_chars": len(fixed), "chars": len(text)})) return out def sentence_line(s: Sentence) -> str: return f"- {s.key} | {' '.join(s.text.split())}\n" def render_parts(bundle: Path, wanted: dict[str, list[str]], catalogue: list[str], label: str, part_chars: int = PART_CHARS) -> list[tuple[str, dict[str, Any]]]: return render_pages(read_pages(bundle), wanted, catalogue, label, part_chars) def render_pages(pages: dict[str, Page], wanted: dict[str, list[str]], catalogue: list[str], label: str, part_chars: int = PART_CHARS) -> list[tuple[str, dict[str, Any]]]: sections = [] for rel, keys in wanted.items(): if rel not in pages: raise RunError(f"{rel} is not a concept page in the bundle") by_key = {s.key: s for s in body_sentences(pages[rel].body)} missing = [k for k in keys if k not in by_key] if missing: raise RunError(f"{rel}: sentence keys not on the page: {', '.join(missing[:3])}") sections.append(Section(rel, [sentence_line(by_key[k]) for k in keys], rel)) return render_sections(label, catalogue, sections, pages, part_chars) def write_parts(out: Path, parts: list[tuple[str, dict[str, Any]]], extra: dict[str, Any] | None = None) -> None: paths = [out / str(stats["file"]) for _, stats in parts] + [out / "parts.json"] if any(p.exists() for p in paths): raise RunError(f"task parts exist in {out}; refusing to overwrite") for (text, _), p in zip(parts, paths): write_new(p, text) write_new(out / "parts.json", dump({"schema": PARTS_SCHEMA, **(extra or {}), "parts": [s for _, s in parts]})) def read_ids(path: Path) -> list[str]: ids: list[str] = [] for line in path.read_text(encoding="utf-8").splitlines(): cid = line.strip().removesuffix(".md") if cid and cid not in ids: ids.append(cid) return ids # --- the guard ------------------------------------------------------------------------------------------------- def kilde_path(text: str) -> str | None: """The first K1 path (kilde/...) in text, with or without a leading /.""" m = KILDE_PATH.search(text) return m.group(1) if m else None def homeless(audit_doc: Any) -> tuple[list[str], list[str]]: """(pages with no-main-page-link, K1 paths with main-page-missing) from the auditor's JSON, each once, sorted. The auditor is the only judge (K2 point 17): nothing here counts links. A main-page-missing finding names its source in `file` or, failing that, in `message`; one that names no kilde/ path cannot run.""" findings = audit_doc.get("findings") if isinstance(audit_doc, dict) else None if not isinstance(findings, list) or not all(isinstance(f, dict) for f in findings): raise RunError("the auditor's output has no list of findings") pages: set[str] = set() missing: set[str] = set() for f in findings: if f.get("check") == "no-main-page-link": pages.add(page_rel(str(f.get("file") or ""))) elif f.get("check") == "main-page-missing": path = kilde_path(str(f.get("file") or "")) or kilde_path(str(f.get("message") or "")) if path is None: raise RunError(f"a main-page-missing finding names no kilde/ source: {f!r}") missing.add(path) return sorted(pages), sorted(missing) def guard_candidates(bundle: Path, sources: list[str]) -> dict[str, list[str]]: """Per K1 path, the ids of the pages that cite it: the only pages that can be its main page (K2 point 1).""" pages = read_pages(bundle) return {src: sorted(concept_id(rel) for rel, p in pages.items() if "/" + src in p.resources.values()) for src in sources} def guard_line(cid: str, pages: dict[str, Page], described: bool) -> str: rel = page_rel(cid) if rel not in pages: raise RunError(f"{rel} is not a concept page in the bundle") p = pages[rel] return f"- {cid} | {' '.join(p.title.split())}" + (f" | {p.description}" if described and p.description else "") + "\n" def guard_parts(bundle: Path, homeless_pages: list[str], sources: list[str], catalogue: list[str], label: str, part_chars: int = PART_CHARS) -> list[str]: """The guard's task: the contract and the catalogue of main pages in every part, then the pages without a home and the sources without a main page (each with the pages that cite it), in parts of at most part_chars.""" pages = read_pages(bundle) candidates = guard_candidates(bundle, sources) head = (f"# Home task: {label}\n\n" + GUARD_CONTRACT + "\n## Catalogue (main pages)\n\n" + "".join(guard_line(cid, pages, False) for cid in catalogue) + "\n") items: list[tuple[str, str]] = [("pages", guard_line(concept_id(page_rel(rel)), pages, True)) for rel in homeless_pages] for src in sources: cands = "".join(guard_line(cid, pages, True) for cid in candidates[src]) or "(no page cites it)\n" items.append(("sources", f"### {src}\n\n{cands}\n")) titles = {"pages": "## Pages without a home\n\n", "sources": "## Sources without a main page\n\n"} def text(group: list[tuple[str, str]]) -> str: out, kind = head, "" for k, piece in group: if k != kind: out, kind = out + ("\n" if kind else "") + titles[k], k out += piece return out room = part_chars - PART_HEADER_ROOM if len(text([])) > room: raise RunError(f"the guard's contract and catalogue leave no room for a page within {part_chars} characters") groups: list[list[tuple[str, str]]] = [] for item in items: if groups and len(text(groups[-1] + [item])) <= room: groups[-1].append(item) elif len(text([item])) <= room: groups.append([item]) else: raise RunError(f"one item of the guard's task does not fit in a part of {part_chars} characters") return [f"\n" + text(g) for i, g in enumerate(groups, 1)] @dataclass class GuardAnswer: homes: list[tuple[str, str]] # (page id, home id) main_pages: list[tuple[str, str]] # (K1 path, page id) none: list[str] unanswered: list[str] rejected: Counter[str] def read_guard(answer: Any, asked_pages: list[str], asked_sources: list[str], catalogue: list[str], candidates: dict[str, list[str]]) -> GuardAnswer: """The guard's answer as code accepts it: per source a main page among the pages that cite it; per page a home from the catalogue or from the main pages this answer names, or none; the first valid answer for each counts.""" doc = answer if isinstance(answer, dict) else {} asked = [concept_id(page_rel(p)) for p in asked_pages] rejected: Counter[str] = Counter() def entries(key: str) -> list[Any]: value = doc.get(key) return value if isinstance(value, list) else [] main_pages: list[tuple[str, str]] = [] for m in entries("main_pages"): if not (isinstance(m, dict) and isinstance(m.get("source"), str) and isinstance(m.get("page"), str)): rejected["malformed"] += 1 continue src, pid = kilde_path(m["source"]), concept_id(page_rel(m["page"])) if src not in asked_sources: rejected["source-not-asked"] += 1 elif any(s == src for s, _ in main_pages): rejected["duplicate"] += 1 elif pid not in candidates.get(src, []): rejected["page-not-a-candidate"] += 1 else: main_pages.append((src, pid)) named = {pid for _, pid in main_pages} targets = set(catalogue) | named homes: list[tuple[str, str]] = [] answered: set[str] = set() for h in entries("homes"): if not (isinstance(h, dict) and isinstance(h.get("page"), str) and isinstance(h.get("home"), str)): rejected["malformed"] += 1 continue pid, hid = concept_id(page_rel(h["page"])), concept_id(page_rel(h["home"])) if pid not in asked: rejected["page-not-asked"] += 1 elif pid in answered: rejected["duplicate"] += 1 elif hid == pid: rejected["self-home"] += 1 elif hid not in targets: rejected["home-not-in-catalogue"] += 1 else: homes.append((pid, hid)) answered.add(pid) none: list[str] = [] for x in entries("none"): if not isinstance(x, str): rejected["malformed"] += 1 continue pid = concept_id(page_rel(x)) if pid not in asked: rejected["page-not-asked"] += 1 elif pid in answered: rejected["duplicate"] += 1 else: none.append(pid) answered.add(pid) unanswered = [pid for pid in asked if pid not in answered and pid not in named] return GuardAnswer(homes, main_pages, none, unanswered, rejected) # --- accept ---------------------------------------------------------------------------------------------------- def find_anchor(s: Sentence, anchor: str) -> tuple[int | None, str | None]: """(offset of the first occurrence outside code, None) or (None, reason).""" if not anchor or any(c in anchor for c in "[]\n"): return None, "anchor-not-in-sentence" in_code = False start = s.text.find(anchor) while start >= 0: end = start + len(anchor) whole = (start == 0 or not s.text[start - 1].isalnum()) and (end == len(s.text) or not s.text[end].isalnum()) if whole: if s.masked[start:end] == anchor: return start, None in_code = True start = s.text.find(anchor, start + 1) return None, "anchor-in-code" if in_code else "anchor-not-in-sentence" def accept(proposals: dict[str, Any], bundle: Path, links: Path, origin: str) -> dict[str, Any]: pages = read_pages(bundle) sentences = {rel: {s.key: s for s in body_sentences(p.body)} for rel, p in pages.items()} doc: dict[str, Any] = load_json(links) if links.is_file() else { "schema": SCHEMA, "links": [], "none": [], "none_count": 0, "accepted": 0, "rejected": {}} records: list[dict[str, Any]] = doc["links"] rejected: Counter[str] = Counter() accepted = 0 raw = proposals.get("links") if isinstance(proposals, dict) else None for p in raw if isinstance(raw, list) else []: reason, record = judge(p, pages, sentences, records) if reason: rejected[reason] += 1 else: assert record is not None records.append(record) accepted += 1 none = proposals.get("none") if isinstance(proposals, dict) else None nones = [str(x) for x in none] if isinstance(none, list) else [] doc["none"] += [{"id": x, "from": origin} for x in nones] doc["none_count"] = len(doc["none"]) doc["accepted"] = len(records) total = Counter(doc["rejected"]) total.update(rejected) doc["rejected"] = {r: total[r] for r in REASONS if total[r]} links.parent.mkdir(parents=True, exist_ok=True) links.write_text(dump(doc), encoding="utf-8") return {"accepted": accepted, "rejected": dict(rejected), "none": len(nones)} def judge(p: Any, pages: dict[str, Page], sentences: dict[str, dict[str, Sentence]], records: list[dict[str, Any]]) -> tuple[str | None, dict[str, Any] | None]: if not (isinstance(p, dict) and all(isinstance(p.get(k), str) for k in ("page", "sentence", "anchor", "target"))): return "malformed", None rel, target, anchor, key = page_rel(p["page"]), page_rel(p["target"]), p["anchor"], p["sentence"] s = sentences.get(rel, {}).get(key) if s is None: return "sentence-missing", None if target not in pages: return "target-missing", None if target == rel: return "self-link", None if not s.footnoted: return "no-footnote", None at, reason = find_anchor(s, anchor) if at is None: return reason, None here = [r for r in records if r["page"] == rel and r["sentence"] == key] if any(r["target"] == target for r in here): return "duplicate", None if len(here) >= MAX_LINKS: return "over-limit", None if any(at < int(r["at"]) + len(str(r["anchor"])) and int(r["at"]) < at + len(anchor) for r in here): return "overlap", None return None, {"page": rel, "sentence": key, "anchor": anchor, "at": at, "target": target} # --- command line ---------------------------------------------------------------------------------------------- def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) s = sub.add_parser("strip") s.add_argument("bundle") s.add_argument("--out", required=True) se = sub.add_parser("sentences") for name in ("--accepted", "--bundle", "--out"): se.add_argument(name, required=True) se.add_argument("--pack", type=int, required=True) t = sub.add_parser("task") t.add_argument("bundle") for name in ("--sentences", "--catalogue", "--out"): t.add_argument(name, required=True) a = sub.add_parser("accept") a.add_argument("proposals") a.add_argument("--bundle", required=True) a.add_argument("--links", required=True) pl = sub.add_parser("place") pl.add_argument("bundle") pl.add_argument("--links", required=True) pl.add_argument("--out", required=True) pl.add_argument("--replace", action="store_true") b = sub.add_parser("pass-before") for name in ("--accepted", "--bundle", "--out"): b.add_argument(name, required=True) b.add_argument("--catalogue") b.add_argument("--pack", type=int, required=True) b.add_argument("--to", type=int) f = sub.add_parser("pass-after") for name in ("--pass-dir", "--bundle", "--links", "--linked", "--sources", "--transcripts", "--okter", "--work", "--kost", "--lines", "--unit"): f.add_argument(name, required=True) for name in ("--cap", "--done", "--total"): f.add_argument(name, type=int, required=True) f.add_argument("--blocks") f.add_argument("--name") return p def sentences_doc(accepted: Path, pack: int, bundle: Path) -> dict[str, Any]: wanted = pack_sentences(load_json(accepted), pack, read_pages(bundle)) return {"schema": SENTENCES_SCHEMA, "pack": pack, "pages": wanted} def pass_before(args: argparse.Namespace) -> int: accepted, pages = load_json(Path(args.accepted)), read_pages(Path(args.bundle)) given = read_ids(Path(args.catalogue)) if args.catalogue else None last = args.pack if args.to is None else args.to if last < args.pack: raise RunError(f"--to {last} is before --pack {args.pack}") jobs = [] for pack in range(args.pack, last + 1): # render every pack before writing any, so a failure leaves nothing behind out = Path(args.out) if args.to is None else Path(args.out) / f"p{pack}" wanted = pack_sentences(accepted, pack, pages) pool = given if given is not None else pack_pool(accepted, pack, pages) ids = filter_catalogue(accepted, pack, pool, pages) jobs.append((pack, out, wanted, pool, ids, render_pages(pages, wanted, ids, f"pack {pack}"))) empty = 0 for pack, out, wanted, pool, ids, parts in jobs: write_new(out / "sentences.json", dump({"schema": SENTENCES_SCHEMA, "pack": pack, "pages": wanted})) write_parts(out, parts, {"pack": pack, "catalogue": {"concepts": len(ids), "pool": len(pool)}}) n = sum(len(v) for v in wanted.values()) empty += not n cat = parts[0][1]["catalogue_chars"] if parts else 0 print(f"pass-before: pack {pack}, pages {len(wanted)}, sentences {n}, parts {len(parts)}, catalogue " f"{len(ids)} of {len(pool)} concepts, {cat} characters -> {out}") for _, stats in parts: print(f" {out / str(stats['file'])} -> {out / ('proposals-' + str(stats['part']) + '.json')}") return 1 if empty else 0 def counts(rejected: dict[str, int]) -> str: return ", ".join(f"{r} {n}" for r, n in sorted(rejected.items())) or "none" def pack_dirs(pass_dir: Path) -> list[Path]: """The p directories of an interval, in pack order.""" dirs = [d for d in pass_dir.iterdir() if d.is_dir() and re.fullmatch(r"p\d+", d.name)] return sorted(dirs, key=lambda d: int(d.name[1:])) def accept_dir(directory: Path, bundle: Path, links: Path) -> tuple[int, int, Counter[str]]: files = sorted(directory.glob("proposals-*.json")) if not files: raise RunError(f"no proposals-*.json in {directory}") totals: Counter[str] = Counter() accepted = nones = 0 for path in files: r = accept(load_json(path), bundle, links, str(path)) accepted, nones = accepted + int(r["accepted"]), nones + int(r["none"]) totals.update(r["rejected"]) return accepted, nones, totals def stop_blocks(kost: dict[str, Any], packs_log: dict[str, Any], lines: list[str], last: int, done: int) -> tuple[int, list[int], int, int | None]: """The blocks done after a pass whose last pack is `last`: the packs up to `last` and every pack with a compiler run in the harvest, so the measured cost and the blocks cover the same packs (order 22 session B2: the next pack's compiler counted over the pass's blocks alone). `done` must be the blocks of the accepted packs up to `last`. Second: the packs with a compiler run in this ledger, over whose blocks the stop line takes the next pack's share (order 27 A3: packs built in an earlier ledger cost nothing here). Third: the blocks of the compiled packs not yet accepted, whose compiler may run once more before the next check. Fourth: their characters (`block_chars`), None when one of them has none (a log written before blocks had characters; motor B17).""" blocks = {int(p["pack"]): int(p["blocks"]) for p in packs_log["packs"]} covered = sum(int(p["blocks"]) for p in packs_log["packs"] if "accepted" in p and int(p["pack"]) <= last) if done != covered: raise RunError(f"--done {done} is not the blocks of the accepted packs up to pack {last} ({covered})") compiled: set[int] = set() for r in kost["runs"]: if r["line"] not in lines: continue if r.get("pack") is None: raise RunError(f"the run {r['description']!r} is tied to no pack: STOP") if int(r["pack"]) not in blocks: raise RunError(f"the run {r['description']!r} is for pack {r['pack']}, which the pack log lacks: STOP") if r["line"] == "build": compiled.add(int(r["pack"])) built = {n for n in blocks if n <= last} | compiled waiting = [p for p in packs_log["packs"] if int(p["pack"]) in built and "accepted" not in p] pending = sum(int(p["blocks"]) for p in waiting) chars = (sum(int(p["block_chars"]) for p in waiting) if all(isinstance(p.get("block_chars"), int) for p in waiting) else None) return sum(blocks[n] for n in built), sorted(compiled), pending, chars def pass_after(args: argparse.Namespace) -> int: pass_dir, bundle, links = Path(args.pass_dir), Path(args.bundle), Path(args.links) dirs = pack_dirs(pass_dir) for d in dirs: # every pack has its proposals before any is accepted, so a missing one changes nothing if not any(d.glob("proposals-*.json")): raise RunError(f"no proposals-*.json in {d}") totals: Counter[str] = Counter() accepted = nones = 0 for d in dirs or [pass_dir]: a, n, t = accept_dir(d, bundle, links) if dirs: print(f"accept {d.name}: accepted {a}, rejected {sum(t.values())} ({counts(dict(t))}), none {n}") accepted, nones = accepted + a, nones + n totals.update(t) print(f"accept: accepted {accepted}, rejected {sum(totals.values())} ({counts(dict(totals))}), none {nones}") placed, stale = place(bundle, links, Path(args.linked), replace=True) print(f"place: {placed} links placed, {stale} stale") if stale: return 1 report = audit.link_rule_report(audit.load_bundle(Path(args.linked), Path(args.sources))) print(f"link rule: {report['pass']} of {report['of']}, misplaced {report['misplaced']}, broken {report['broken']}, " f"max out-degree {report['max_out_degree']}") harvest = ["harvest", "--transcripts", args.transcripts, "--okter", args.okter, "--work", args.work, "--out", args.kost] code = cost.main(harvest) print(f"cost harvest: exit {code}") if code: return code done, following, chars = args.done, None, None if args.unit == "blocks": if not args.blocks: raise RunError("--unit blocks needs --blocks (the pack log)") last = max(int(d.name[1:]) for d in dirs) if dirs else int(load_json(pass_dir / "sentences.json")["pack"]) done, packs, following, chars = stop_blocks(load_json(Path(args.kost)), load_json(Path(args.blocks)), cost.capped_lines(args.lines), last, args.done) print(f"done: {done} of {args.total} blocks (packs {', '.join(map(str, packs))} with a compiler run)") stop = ["stop", "--lines", args.lines, "--cap", str(args.cap), "--unit", args.unit, "--done", str(done), "--total", str(args.total), "--kost", args.kost] stop += ["--blocks", args.blocks] if args.blocks else [] stop += ["--name", args.name] if args.name else [] stop += ["--next", str(following)] if following is not None else [] stop += ["--next-chars", str(chars)] if following is not None and chars is not None else [] code = cost.main(stop) print(f"cost stop: exit {code}") return code def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: if args.command == "strip": n = strip(Path(args.bundle), Path(args.out)) print(f"strip: {n} links reduced to their anchor text -> {args.out}") return 0 if args.command == "sentences": doc = sentences_doc(Path(args.accepted), args.pack, Path(args.bundle)) write_new(Path(args.out), dump(doc)) n = sum(len(v) for v in doc["pages"].values()) print(f"sentences: pack {args.pack}, {len(doc['pages'])} pages, {n} sentences -> {args.out}") return 0 if n else 1 if args.command == "pass-before": return pass_before(args) if args.command == "task": out = Path(args.out) doc = load_json(Path(args.sentences)) label = Path(args.sentences).stem parts = render_parts(Path(args.bundle), doc["pages"], read_ids(Path(args.catalogue)), label) write_parts(out, parts) n = sum(len(v) for v in doc["pages"].values()) cat = parts[0][1]["catalogue_chars"] if parts else 0 print(f"{args.command}: pages {len(doc['pages'])}, sentences {n}, parts {len(parts)}, catalogue {cat} " f"characters -> {out}") for _, stats in parts: print(f" {out / str(stats['file'])} -> {out / ('proposals-' + str(stats['part']) + '.json')}") return 0 if n else 1 if args.command == "accept": r = accept(load_json(Path(args.proposals)), Path(args.bundle), Path(args.links), args.proposals) print(f"accept: accepted {r['accepted']}, rejected {sum(r['rejected'].values())} " f"({counts(r['rejected'])}), none {r['none']}") return 0 if args.command == "place": placed, stale = place(Path(args.bundle), Path(args.links), Path(args.out), args.replace) print(f"place: {placed} links placed, {stale} stale -> {args.out}") return 1 if stale else 0 return pass_after(args) except (RunError, OSError, json.JSONDecodeError, KeyError, ValueError, TypeError) as exc: print(f"klink cannot run: {exc}", file=sys.stderr) return 2 if __name__ == "__main__": sys.exit(main())