First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
957 lines
45 KiB
Python
957 lines
45 KiB
Python
"""The link pass: the model proposes links by meaning; this code strips, asks, accepts and places them. No model call.
|
|
|
|
Usage:
|
|
python3 -m okfwiki.klink strip BUNDLE --out DIR
|
|
python3 -m okfwiki.klink sentences --accepted FILE --pack N --bundle DIR --out FILE
|
|
python3 -m okfwiki.klink task BUNDLE --sentences FILE --catalogue IDS --out DIR
|
|
python3 -m okfwiki.klink accept PROPOSALS --bundle DIR --links FILE
|
|
python3 -m okfwiki.klink place BUNDLE --links FILE --out DIR [--replace]
|
|
python3 -m okfwiki.klink pass-before --accepted FILE --pack N [--to M] --bundle DIR [--catalogue IDS] --out DIR
|
|
python3 -m okfwiki.klink pass-after --pass-dir DIR --bundle DIR --links FILE --linked DIR --sources DIR
|
|
--transcripts DIR --okter FILE --work DIR --kost FILE --lines L[,L] --cap N --unit U --done D --total T
|
|
[--blocks FILE] [--name NAME]
|
|
|
|
Links are data (`links.json`, schema okf-wiki/links/1), never edits to pages, because K re-assembles pages from
|
|
fragments. A record is {page, sentence, anchor, at, target}: the sentence key, the anchor text and its character
|
|
offset in that sentence, and the target page. The sentence key is the first 16 hex characters of the sha256 of the
|
|
sentence's original text (sliced with audit.sentence_spans from the unmasked text) after footnote marks are removed,
|
|
every markdown link is reduced to its anchor text and whitespace is collapsed, plus "-<n>", the occurrence of that
|
|
text on the page. Every command first reduces each bundle link in a concept body to its anchor text, in prose, code,
|
|
headings and footnote definitions alike, so keys and offsets are computed on link-free text and placement is
|
|
deterministic. index.md and log.md are copied unchanged.
|
|
|
|
strip copies the bundle with every bundle link in a concept body reduced to its anchor text.
|
|
sentences the sentence keys on assembled pages that come from pack N's accepted fragments (`pack`, `concept_id`,
|
|
`text`). Exit 1 when the pack has no sentence (an empty denominator).
|
|
task renders the task parts: the contract, the catalogue as `id | title` lines and, per page, only the listed
|
|
sentences with their keys; parts are capped at 60,000 characters. A catalogue that alone exceeds the cap
|
|
cannot run (exit 2): it has to be split or filtered first.
|
|
accept validates proposals {"links": [{page, sentence, anchor, target}], "none": [ids]} and appends accepted
|
|
records; rejects are counted per reason, `none` answers are counted, and no record is ever rewritten.
|
|
place writes every record into a copy of the bundle; refuses an existing output unless --replace (then it writes
|
|
a temporary directory and swaps it in). A record whose sentence no longer exists is stale: exit 1.
|
|
pass-before / pass-after chain the non-model steps of one pass (sentences -> task; accept -> place -> link rule ->
|
|
cost harvest -> cost stop) and print only counts, so a pass costs the orchestrator few turns.
|
|
With --to M, pass-before covers packs N..M, one directory p<n> per pack, and pass-after on the parent
|
|
accepts every p<n> in pack order: the same links as one pass per pack, for the same orchestrator turns.
|
|
With --unit blocks, --done is the blocks of the accepted packs the pass covers (the done line of run-after
|
|
for its last pack), and the stop line divides by the blocks of every pack with a compiler run: in the
|
|
skill's chain the compiler of the next pack has run before this pass-after, so the harvest holds its run
|
|
and both sides of the stop line cover the same packs. The orchestration is one line for the whole session
|
|
and is not split per pack. A --done that is not those blocks, or a run on a capped line that is tied to no
|
|
pack or to a pack the log lacks, cannot run (exit 2, STOP). The stop is measured cost plus the blocks of a
|
|
compiled pack not yet accepted (cost stop --next, and --next-chars when the pack log has their characters), and the projection is reported only (the operator
|
|
12.10.2026).
|
|
The catalogue of a pack is filtered to its candidate targets within CATALOGUE_CHARS (filter_catalogue)
|
|
from a pool: the ids in --catalogue, or else the concepts with fragments in this or an earlier pack.
|
|
Exit 0 done, 1 a counted failure (stale, empty, STOP), 2 cannot run.
|
|
|
|
The guard (contract K2 point 17; kpack run-after calls it, it has no command of its own): `homeless` takes the pages
|
|
with a no-main-page-link finding and the sources with a main-page-missing finding from the auditor's JSON, so the
|
|
auditor is the only judge. `guard_parts` renders one task for the model with GUARD_CONTRACT, the pack's filtered
|
|
catalogue of main pages, each page without a home (id, title, description) and each source without a main page with
|
|
the pages that cite it; `read_guard` checks the answer: a home or none per page, a main page per source. The guard's
|
|
task runs on the linker agent, because a home is a link from a main page chosen by meaning.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import posixpath
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
from collections import Counter
|
|
from dataclasses import dataclass
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Any
|
|
|
|
from okfwiki import audit, cost
|
|
from okfwiki.common import RunError, write_new
|
|
|
|
SCHEMA = "okf-wiki/links/1"
|
|
SENTENCES_SCHEMA = "okf-wiki/link-sentences/1"
|
|
PARTS_SCHEMA = "okf-wiki/link-parts/1"
|
|
PART_CHARS = 60_000
|
|
CATALOGUE_CHARS = 32_000 # leaves ~27,000 characters of a part for sentences; order 28 A3: with the main pages first, the
|
|
# fewest far concepts a pack of the rest saw in the model was 205 (24,000: three packs under G's 179)
|
|
RECENT_PACKS = 2
|
|
MAX_LINKS = 3
|
|
PART_HEADER_ROOM = len("<!-- part 999 of 999 -->\n")
|
|
FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})")
|
|
REASONS = ("malformed", "sentence-missing", "target-missing", "self-link", "no-footnote", "anchor-not-in-sentence",
|
|
"anchor-in-code", "duplicate", "over-limit", "overlap")
|
|
|
|
CONTRACT = """## Contract
|
|
|
|
You add links to sentences of a knowledge collection. Every sentence below has a key. Link by meaning, not by name:
|
|
a link says that a phrase in the sentence is about a concept in the catalogue, and the phrase may use other words than
|
|
the concept's title.
|
|
|
|
- Targets: only ids from the catalogue in this task. Never link a page to itself.
|
|
- Anchor: a phrase that already stands, character for character, in the sentence, outside backticks. Change no word.
|
|
- At most 3 links in one sentence. No lists of related pages and no "see also" lines.
|
|
- When nothing in the catalogue fits, put the page id in `none`.
|
|
- Output: one JSON object `{"links": [{"page": ..., "sentence": ..., "anchor": ..., "target": ...}], "none": [...]}`,
|
|
where `page` is the page file as the task names it, `sentence` is the sentence key and `target` is a catalogue id.
|
|
Write it to the output path named in your prompt, then stop.
|
|
"""
|
|
|
|
GUARD_CONTRACT = """## Contract (home task)
|
|
|
|
A main page is the page for the main topic of one docs source. Every other page needs a home: a main page that links
|
|
to it in its list of related concepts. The pages and sources below lack one. Decide by meaning, as a reader of the
|
|
main page would look for the page.
|
|
|
|
- For each page under "Pages without a home": choose its home from the catalogue, or a page you name a main page
|
|
below, and add {"page": <id>, "home": <id>} to `homes`. When no main page fits, put the page id in `none`: a page
|
|
without a natural home is reported, never given a forced home.
|
|
- For each source under "Sources without a main page": choose, from the pages listed under it, the page whose topic is
|
|
that source's main topic, and add {"source": <path>, "page": <id>} to `main_pages`.
|
|
- Output: one JSON object `{"homes": [...], "none": [...], "main_pages": [...]}`. Write it to the output path named
|
|
in your prompt, then stop.
|
|
"""
|
|
GUARDED = frozenset({"no-main-page-link", "main-page-missing"})
|
|
GUARD_REASONS = ("malformed", "page-not-asked", "duplicate", "self-home", "home-not-in-catalogue", "source-not-asked",
|
|
"page-not-a-candidate")
|
|
KILDE_PATH = re.compile(r"(?<![\w.-])/?(kilde/[a-z0-9._-]*[a-z0-9_-])")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Sentence:
|
|
key: str
|
|
text: str # the original slice of the link-reduced body, footnote marks kept
|
|
masked: str # the same slice with inline code masked
|
|
start: int # offset of the slice in the link-reduced body
|
|
footnoted: bool
|
|
labels: tuple[str, ...]
|
|
|
|
|
|
# --- pages -----------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def split_page(text: str) -> tuple[str, str]:
|
|
"""(frontmatter block with its closing fence and newline, body); a page without frontmatter has an empty head."""
|
|
lines = text.split("\n")
|
|
if not lines or lines[0].strip() != "---":
|
|
return "", text
|
|
end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
|
|
if end is None:
|
|
return "", text
|
|
return "\n".join(lines[:end + 1]) + "\n", "\n".join(lines[end + 1:])
|
|
|
|
|
|
def reduce_links(body: str) -> tuple[str, int]:
|
|
"""Every bundle link reduced to its anchor text, wherever it stands in the body; external links are kept."""
|
|
count = 0
|
|
|
|
def one(m: re.Match[str]) -> str:
|
|
nonlocal count
|
|
if not audit.is_bundle_link(m.group(2)):
|
|
return m.group(0)
|
|
count += 1
|
|
return m.group(1)
|
|
|
|
return audit.sub_links(one, body), count
|
|
|
|
|
|
def is_concept(rel: str) -> bool:
|
|
return rel.endswith(".md") and PurePosixPath(rel).name not in audit.RESERVED
|
|
|
|
|
|
def key_text(text: str) -> str:
|
|
plain = audit.sub_links(lambda m: m.group(1), audit.FOOTNOTE_REF.sub("", text))
|
|
return " ".join(plain.split())
|
|
|
|
|
|
def prose_spans(body: str) -> list[tuple[int, int]]:
|
|
"""(start, end) offsets of paragraphs and list items; headings, footnote definitions and fenced blocks are not prose."""
|
|
spans: list[tuple[int, int]] = []
|
|
current: list[int] | None = None
|
|
fence = ""
|
|
offset = 0
|
|
for raw in body.split("\n"):
|
|
start, end = offset, offset + len(raw)
|
|
offset = end + 1
|
|
f = FENCE.match(raw)
|
|
if fence:
|
|
if f and f.group(1)[0] == fence[0] and len(f.group(1)) >= len(fence):
|
|
fence = ""
|
|
continue
|
|
if f:
|
|
fence, current = f.group(1), None
|
|
continue
|
|
if not raw.strip() or audit.HEADING.match(raw) or audit.FOOTNOTE_DEF.match(raw.strip()):
|
|
current = None
|
|
continue
|
|
item = audit.LIST_ITEM.match(raw)
|
|
if item:
|
|
current = [start + item.end(), end]
|
|
spans.append((current[0], current[1]))
|
|
elif current is None:
|
|
current = [start + len(raw) - len(raw.lstrip()), end]
|
|
spans.append((current[0], current[1]))
|
|
else:
|
|
current[1] = end
|
|
spans[-1] = (current[0], current[1])
|
|
return spans
|
|
|
|
|
|
def body_sentences(body: str) -> list[Sentence]:
|
|
"""The sentences of a link-reduced body, each with its key."""
|
|
out: list[Sentence] = []
|
|
seen: Counter[str] = Counter()
|
|
for a, b in prose_spans(body):
|
|
block = body[a:b]
|
|
masked = audit.mask_code(block)
|
|
for s, e in audit.sentence_spans(block):
|
|
text = block[s:e]
|
|
norm = key_text(text)
|
|
digest = hashlib.sha256(norm.encode("utf-8")).hexdigest()[:16]
|
|
labels = tuple(audit.FOOTNOTE_REF.findall(masked[s:e]))
|
|
out.append(Sentence(f"{digest}-{seen[digest]}", text, masked[s:e], a + s, bool(labels), labels))
|
|
seen[digest] += 1
|
|
return out
|
|
|
|
|
|
def page_sentences(text: str) -> list[Sentence]:
|
|
return body_sentences(reduce_links(split_page(text)[1])[0])
|
|
|
|
|
|
@dataclass
|
|
class Page:
|
|
rel: str
|
|
head: str
|
|
body: str # link-reduced
|
|
title: str
|
|
resources: dict[str, str] # sources[].id -> resource
|
|
description: str = ""
|
|
|
|
|
|
def read_pages(bundle: Path) -> dict[str, Page]:
|
|
pages: dict[str, Page] = {}
|
|
for p in sorted(bundle.rglob("*.md")):
|
|
rel = p.relative_to(bundle).as_posix()
|
|
if not is_concept(rel):
|
|
continue
|
|
doc = audit.read_document(p, rel)
|
|
head, body = split_page(p.read_text(encoding="utf-8"))
|
|
fm = doc.frontmatter or {}
|
|
srcs = fm.get("sources")
|
|
resources = {str(s["id"]): str(s.get("resource", "")) for s in (srcs if isinstance(srcs, list) else [])
|
|
if isinstance(s, dict) and "id" in s}
|
|
pages[rel] = Page(rel, head, reduce_links(body)[0], str(fm.get("title") or PurePosixPath(rel).stem), resources,
|
|
" ".join(str(fm.get("description") or "").split()))
|
|
return pages
|
|
|
|
|
|
def page_rel(name: str) -> str:
|
|
name = name.strip()
|
|
return name if name.endswith(".md") else name + ".md"
|
|
|
|
|
|
def load_json(path: Path) -> Any:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def dump(doc: Any) -> str:
|
|
return json.dumps(doc, ensure_ascii=False, indent=1) + "\n"
|
|
|
|
|
|
# --- strip and place -------------------------------------------------------------------------------------------
|
|
|
|
|
|
def copy_reduced(bundle: Path, out: Path, records: list[dict[str, Any]] | None) -> tuple[int, int, int]:
|
|
"""Copy bundle to out with every bundle link reduced, then place records; (links reduced, placed, stale)."""
|
|
by_page: dict[str, list[dict[str, Any]]] = {}
|
|
for r in records or []:
|
|
by_page.setdefault(str(r["page"]), []).append(r)
|
|
reduced = placed = stale = 0
|
|
for p in sorted(bundle.rglob("*")):
|
|
if not p.is_file():
|
|
continue
|
|
rel = p.relative_to(bundle).as_posix()
|
|
dst = out / rel
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
if not is_concept(rel):
|
|
shutil.copyfile(p, dst)
|
|
continue
|
|
head, body = split_page(p.read_text(encoding="utf-8"))
|
|
body, n = reduce_links(body)
|
|
reduced += n
|
|
body, ok, bad = place_records(rel, body, by_page.pop(rel, []))
|
|
placed, stale = placed + ok, stale + bad
|
|
dst.write_text(head + body, encoding="utf-8")
|
|
stale += sum(len(v) for v in by_page.values()) # records for pages that no longer exist
|
|
return reduced, placed, stale
|
|
|
|
|
|
def place_records(rel: str, body: str, records: list[dict[str, Any]]) -> tuple[str, int, int]:
|
|
by_key = {s.key: s for s in body_sentences(body)}
|
|
edits: list[tuple[int, int, str]] = []
|
|
stale = 0
|
|
for r in records:
|
|
s = by_key.get(str(r["sentence"]))
|
|
anchor, at = str(r["anchor"]), int(r["at"])
|
|
if s is None or s.text[at:at + len(anchor)] != anchor:
|
|
stale += 1
|
|
continue
|
|
target = posixpath.relpath(str(r["target"]), posixpath.dirname(rel) or ".")
|
|
edits.append((s.start + at, s.start + at + len(anchor), f"[{anchor}]({target})"))
|
|
for a, b, link in sorted(edits, reverse=True):
|
|
body = body[:a] + link + body[b:]
|
|
return body, len(edits), stale
|
|
|
|
|
|
def strip(bundle: Path, out: Path) -> int:
|
|
if out.exists():
|
|
raise RunError(f"{out} exists; refusing to overwrite")
|
|
return copy_reduced(bundle, out, None)[0]
|
|
|
|
|
|
def place(bundle: Path, links: Path, out: Path, replace: bool) -> tuple[int, int]:
|
|
records = load_json(links)["links"]
|
|
if out.exists() and not replace:
|
|
raise RunError(f"{out} exists; refusing to overwrite without --replace")
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = Path(tempfile.mkdtemp(prefix=f".{out.name}-", dir=out.parent))
|
|
try:
|
|
_, placed, stale = copy_reduced(bundle, tmp, records)
|
|
if out.exists():
|
|
old = out.with_name(f".{out.name}-old")
|
|
out.rename(old)
|
|
tmp.rename(out)
|
|
shutil.rmtree(old)
|
|
else:
|
|
tmp.rename(out)
|
|
finally:
|
|
if tmp.exists():
|
|
shutil.rmtree(tmp)
|
|
return placed, stale
|
|
|
|
|
|
# --- what the model is asked -----------------------------------------------------------------------------------
|
|
|
|
|
|
def pack_sentences(accepted: dict[str, Any], pack: int, pages: dict[str, Page]) -> dict[str, list[str]]:
|
|
texts: dict[str, list[str]] = {}
|
|
for f in accepted["fragments"]:
|
|
if int(f["pack"]) == pack:
|
|
texts.setdefault(page_rel(str(f["concept_id"])), []).append(key_text(str(f["text"])))
|
|
out: dict[str, list[str]] = {}
|
|
for rel, frags in texts.items():
|
|
if rel not in pages:
|
|
continue
|
|
keys = [s.key for s in body_sentences(pages[rel].body)
|
|
if s.footnoted and any(key_text(s.text).rstrip(".!?") in t for t in frags)]
|
|
if keys:
|
|
out[rel] = keys
|
|
return out
|
|
|
|
|
|
def concept_id(rel: str) -> str:
|
|
return PurePosixPath(rel).with_suffix("").as_posix()
|
|
|
|
|
|
def pack_pool(accepted: dict[str, Any], pack: int, pages: dict[str, Page]) -> list[str]:
|
|
"""The concepts with fragments in this or an earlier pack: the pages that exist after pack N in an additive build."""
|
|
ids = {concept_id(page_rel(str(f["concept_id"]))) for f in accepted["fragments"] if int(f["pack"]) <= pack}
|
|
return sorted(cid for cid in ids if page_rel(cid) in pages)
|
|
|
|
|
|
def filter_catalogue(accepted: dict[str, Any], pack: int, pool: list[str], pages: dict[str, Page],
|
|
budget: int = CATALOGUE_CHARS) -> list[str]:
|
|
"""The candidate targets of a pack, ranked, cut at the first that does not fit the character budget.
|
|
|
|
The main pages (accepted.json's main_pages) come first, then the rest, each in rank order (order 28 A3: they are
|
|
the homes of criterion 6, and with the old rule a late pack of collection 1 saw few of them). Rank: (0) the
|
|
concepts this pack writes to; (1) those last written to in the RECENT_PACKS packs before it, latest first, because
|
|
the fixed order puts related sources next to each other; (2) the rest by how many distinct source files their page
|
|
cites, most first, so a late pack can still reach the central concepts made early. Ties by id. The rule uses
|
|
structure only (packs, citations and main pages), never names: the model links by meaning.
|
|
"""
|
|
last: dict[str, int] = {}
|
|
for f in accepted["fragments"]:
|
|
p, cid = int(f["pack"]), concept_id(page_rel(str(f["concept_id"])))
|
|
if p <= pack:
|
|
last[cid] = max(last.get(cid, 0), p)
|
|
|
|
def rank(cid: str) -> tuple[int, int, str]:
|
|
lp = last.get(cid)
|
|
if lp == pack:
|
|
return 0, 0, cid
|
|
if lp is not None and pack - lp <= RECENT_PACKS:
|
|
return 1, -lp, cid
|
|
return 2, -len(set(pages[page_rel(cid)].resources.values()) - {""}), cid
|
|
|
|
mains = {concept_id(page_rel(str(m["concept_id"]))) for m in accepted.get("main_pages", [])}
|
|
ranked = sorted(pool, key=rank)
|
|
out: list[str] = []
|
|
used = 0
|
|
for cid in [c for c in ranked if c in mains] + [c for c in ranked if c not in mains]:
|
|
size = len(catalogue_text([cid], pages))
|
|
if used + size > budget:
|
|
break
|
|
out.append(cid)
|
|
used += size
|
|
return out
|
|
|
|
|
|
@dataclass
|
|
class Section:
|
|
page: str
|
|
lines: list[str]
|
|
heading: str
|
|
|
|
|
|
def catalogue_text(ids: list[str], pages: dict[str, Page]) -> str:
|
|
out = []
|
|
for cid in ids:
|
|
rel = page_rel(cid)
|
|
if rel not in pages:
|
|
raise RunError(f"catalogue id {cid!r} is not a concept page in the bundle")
|
|
out.append(f"{PurePosixPath(rel).with_suffix('').as_posix()} | {pages[rel].title}\n")
|
|
return "".join(out)
|
|
|
|
|
|
def part_text(label: str, catalogue: str, sections: list[Section]) -> str:
|
|
body = "".join(f"### {s.heading}\n" + "".join(s.lines) + "\n" for s in sections)
|
|
return (f"# Link pass: {label}\n\n" + CONTRACT + "\n## Catalogue (link targets)\n\n" + catalogue
|
|
+ "\n## Sentences\n\n" + body)
|
|
|
|
|
|
def render_sections(label: str, fixed_ids: list[str], sections: list[Section], pages: dict[str, Page],
|
|
part_chars: int) -> list[tuple[str, dict[str, Any]]]:
|
|
fixed = catalogue_text(fixed_ids, pages)
|
|
if len(fixed) > part_chars:
|
|
raise RunError(f"the catalogue alone is {len(fixed)} characters ({len(fixed_ids)} concepts), over the part "
|
|
f"limit of {part_chars}: split or filter the catalogue before this pass")
|
|
room = part_chars - PART_HEADER_ROOM
|
|
|
|
def size(group: list[Section]) -> int:
|
|
return len(part_text(label, fixed, group))
|
|
|
|
if size([]) > room:
|
|
raise RunError(f"the contract and catalogue leave no room for a sentence within {part_chars} characters")
|
|
chunks: list[Section] = []
|
|
for s in sections:
|
|
piece = Section(s.page, [], s.heading)
|
|
for line in s.lines:
|
|
if piece.lines and size([Section(s.page, piece.lines + [line], piece.heading)]) > room:
|
|
chunks.append(piece)
|
|
piece = Section(s.page, [], s.heading + " (continued)")
|
|
piece.lines.append(line)
|
|
if size([piece]) > room:
|
|
raise RunError(f"one sentence of {s.page} does not fit in a part of {part_chars} characters")
|
|
chunks.append(piece)
|
|
groups: list[list[Section]] = []
|
|
for c in chunks:
|
|
if groups and size(groups[-1] + [c]) <= room:
|
|
groups[-1].append(c)
|
|
else:
|
|
groups.append([c])
|
|
out: list[tuple[str, dict[str, Any]]] = []
|
|
for i, group in enumerate(groups, 1):
|
|
text = f"<!-- part {i} of {len(groups)} -->\n" + part_text(label, fixed, group)
|
|
out.append((text, {"part": i, "file": f"part-{i}.md", "pages": len({s.page for s in group}),
|
|
"sentences": sum(len(s.lines) for s in group), "catalogue_chars": len(fixed),
|
|
"chars": len(text)}))
|
|
return out
|
|
|
|
|
|
def sentence_line(s: Sentence) -> str:
|
|
return f"- {s.key} | {' '.join(s.text.split())}\n"
|
|
|
|
|
|
def render_parts(bundle: Path, wanted: dict[str, list[str]], catalogue: list[str], label: str,
|
|
part_chars: int = PART_CHARS) -> list[tuple[str, dict[str, Any]]]:
|
|
return render_pages(read_pages(bundle), wanted, catalogue, label, part_chars)
|
|
|
|
|
|
def render_pages(pages: dict[str, Page], wanted: dict[str, list[str]], catalogue: list[str], label: str,
|
|
part_chars: int = PART_CHARS) -> list[tuple[str, dict[str, Any]]]:
|
|
sections = []
|
|
for rel, keys in wanted.items():
|
|
if rel not in pages:
|
|
raise RunError(f"{rel} is not a concept page in the bundle")
|
|
by_key = {s.key: s for s in body_sentences(pages[rel].body)}
|
|
missing = [k for k in keys if k not in by_key]
|
|
if missing:
|
|
raise RunError(f"{rel}: sentence keys not on the page: {', '.join(missing[:3])}")
|
|
sections.append(Section(rel, [sentence_line(by_key[k]) for k in keys], rel))
|
|
return render_sections(label, catalogue, sections, pages, part_chars)
|
|
|
|
|
|
def write_parts(out: Path, parts: list[tuple[str, dict[str, Any]]], extra: dict[str, Any] | None = None) -> None:
|
|
paths = [out / str(stats["file"]) for _, stats in parts] + [out / "parts.json"]
|
|
if any(p.exists() for p in paths):
|
|
raise RunError(f"task parts exist in {out}; refusing to overwrite")
|
|
for (text, _), p in zip(parts, paths):
|
|
write_new(p, text)
|
|
write_new(out / "parts.json", dump({"schema": PARTS_SCHEMA, **(extra or {}), "parts": [s for _, s in parts]}))
|
|
|
|
|
|
def read_ids(path: Path) -> list[str]:
|
|
ids: list[str] = []
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
cid = line.strip().removesuffix(".md")
|
|
if cid and cid not in ids:
|
|
ids.append(cid)
|
|
return ids
|
|
|
|
|
|
# --- the guard -------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def kilde_path(text: str) -> str | None:
|
|
"""The first K1 path (kilde/...) in text, with or without a leading /."""
|
|
m = KILDE_PATH.search(text)
|
|
return m.group(1) if m else None
|
|
|
|
|
|
def homeless(audit_doc: Any) -> tuple[list[str], list[str]]:
|
|
"""(pages with no-main-page-link, K1 paths with main-page-missing) from the auditor's JSON, each once, sorted.
|
|
|
|
The auditor is the only judge (K2 point 17): nothing here counts links. A main-page-missing finding names its
|
|
source in `file` or, failing that, in `message`; one that names no kilde/ path cannot run."""
|
|
findings = audit_doc.get("findings") if isinstance(audit_doc, dict) else None
|
|
if not isinstance(findings, list) or not all(isinstance(f, dict) for f in findings):
|
|
raise RunError("the auditor's output has no list of findings")
|
|
pages: set[str] = set()
|
|
missing: set[str] = set()
|
|
for f in findings:
|
|
if f.get("check") == "no-main-page-link":
|
|
pages.add(page_rel(str(f.get("file") or "")))
|
|
elif f.get("check") == "main-page-missing":
|
|
path = kilde_path(str(f.get("file") or "")) or kilde_path(str(f.get("message") or ""))
|
|
if path is None:
|
|
raise RunError(f"a main-page-missing finding names no kilde/ source: {f!r}")
|
|
missing.add(path)
|
|
return sorted(pages), sorted(missing)
|
|
|
|
|
|
def guard_candidates(bundle: Path, sources: list[str]) -> dict[str, list[str]]:
|
|
"""Per K1 path, the ids of the pages that cite it: the only pages that can be its main page (K2 point 1)."""
|
|
pages = read_pages(bundle)
|
|
return {src: sorted(concept_id(rel) for rel, p in pages.items() if "/" + src in p.resources.values())
|
|
for src in sources}
|
|
|
|
|
|
def guard_line(cid: str, pages: dict[str, Page], described: bool) -> str:
|
|
rel = page_rel(cid)
|
|
if rel not in pages:
|
|
raise RunError(f"{rel} is not a concept page in the bundle")
|
|
p = pages[rel]
|
|
return f"- {cid} | {' '.join(p.title.split())}" + (f" | {p.description}" if described and p.description else "") + "\n"
|
|
|
|
|
|
def guard_parts(bundle: Path, homeless_pages: list[str], sources: list[str], catalogue: list[str], label: str,
|
|
part_chars: int = PART_CHARS) -> list[str]:
|
|
"""The guard's task: the contract and the catalogue of main pages in every part, then the pages without a home
|
|
and the sources without a main page (each with the pages that cite it), in parts of at most part_chars."""
|
|
pages = read_pages(bundle)
|
|
candidates = guard_candidates(bundle, sources)
|
|
head = (f"# Home task: {label}\n\n" + GUARD_CONTRACT + "\n## Catalogue (main pages)\n\n"
|
|
+ "".join(guard_line(cid, pages, False) for cid in catalogue) + "\n")
|
|
items: list[tuple[str, str]] = [("pages", guard_line(concept_id(page_rel(rel)), pages, True))
|
|
for rel in homeless_pages]
|
|
for src in sources:
|
|
cands = "".join(guard_line(cid, pages, True) for cid in candidates[src]) or "(no page cites it)\n"
|
|
items.append(("sources", f"### {src}\n\n{cands}\n"))
|
|
titles = {"pages": "## Pages without a home\n\n", "sources": "## Sources without a main page\n\n"}
|
|
|
|
def text(group: list[tuple[str, str]]) -> str:
|
|
out, kind = head, ""
|
|
for k, piece in group:
|
|
if k != kind:
|
|
out, kind = out + ("\n" if kind else "") + titles[k], k
|
|
out += piece
|
|
return out
|
|
|
|
room = part_chars - PART_HEADER_ROOM
|
|
if len(text([])) > room:
|
|
raise RunError(f"the guard's contract and catalogue leave no room for a page within {part_chars} characters")
|
|
groups: list[list[tuple[str, str]]] = []
|
|
for item in items:
|
|
if groups and len(text(groups[-1] + [item])) <= room:
|
|
groups[-1].append(item)
|
|
elif len(text([item])) <= room:
|
|
groups.append([item])
|
|
else:
|
|
raise RunError(f"one item of the guard's task does not fit in a part of {part_chars} characters")
|
|
return [f"<!-- part {i} of {len(groups)} -->\n" + text(g) for i, g in enumerate(groups, 1)]
|
|
|
|
|
|
@dataclass
|
|
class GuardAnswer:
|
|
homes: list[tuple[str, str]] # (page id, home id)
|
|
main_pages: list[tuple[str, str]] # (K1 path, page id)
|
|
none: list[str]
|
|
unanswered: list[str]
|
|
rejected: Counter[str]
|
|
|
|
|
|
def read_guard(answer: Any, asked_pages: list[str], asked_sources: list[str], catalogue: list[str],
|
|
candidates: dict[str, list[str]]) -> GuardAnswer:
|
|
"""The guard's answer as code accepts it: per source a main page among the pages that cite it; per page a home
|
|
from the catalogue or from the main pages this answer names, or none; the first valid answer for each counts."""
|
|
doc = answer if isinstance(answer, dict) else {}
|
|
asked = [concept_id(page_rel(p)) for p in asked_pages]
|
|
rejected: Counter[str] = Counter()
|
|
|
|
def entries(key: str) -> list[Any]:
|
|
value = doc.get(key)
|
|
return value if isinstance(value, list) else []
|
|
|
|
main_pages: list[tuple[str, str]] = []
|
|
for m in entries("main_pages"):
|
|
if not (isinstance(m, dict) and isinstance(m.get("source"), str) and isinstance(m.get("page"), str)):
|
|
rejected["malformed"] += 1
|
|
continue
|
|
src, pid = kilde_path(m["source"]), concept_id(page_rel(m["page"]))
|
|
if src not in asked_sources:
|
|
rejected["source-not-asked"] += 1
|
|
elif any(s == src for s, _ in main_pages):
|
|
rejected["duplicate"] += 1
|
|
elif pid not in candidates.get(src, []):
|
|
rejected["page-not-a-candidate"] += 1
|
|
else:
|
|
main_pages.append((src, pid))
|
|
named = {pid for _, pid in main_pages}
|
|
targets = set(catalogue) | named
|
|
homes: list[tuple[str, str]] = []
|
|
answered: set[str] = set()
|
|
for h in entries("homes"):
|
|
if not (isinstance(h, dict) and isinstance(h.get("page"), str) and isinstance(h.get("home"), str)):
|
|
rejected["malformed"] += 1
|
|
continue
|
|
pid, hid = concept_id(page_rel(h["page"])), concept_id(page_rel(h["home"]))
|
|
if pid not in asked:
|
|
rejected["page-not-asked"] += 1
|
|
elif pid in answered:
|
|
rejected["duplicate"] += 1
|
|
elif hid == pid:
|
|
rejected["self-home"] += 1
|
|
elif hid not in targets:
|
|
rejected["home-not-in-catalogue"] += 1
|
|
else:
|
|
homes.append((pid, hid))
|
|
answered.add(pid)
|
|
none: list[str] = []
|
|
for x in entries("none"):
|
|
if not isinstance(x, str):
|
|
rejected["malformed"] += 1
|
|
continue
|
|
pid = concept_id(page_rel(x))
|
|
if pid not in asked:
|
|
rejected["page-not-asked"] += 1
|
|
elif pid in answered:
|
|
rejected["duplicate"] += 1
|
|
else:
|
|
none.append(pid)
|
|
answered.add(pid)
|
|
unanswered = [pid for pid in asked if pid not in answered and pid not in named]
|
|
return GuardAnswer(homes, main_pages, none, unanswered, rejected)
|
|
|
|
|
|
# --- accept ----------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def find_anchor(s: Sentence, anchor: str) -> tuple[int | None, str | None]:
|
|
"""(offset of the first occurrence outside code, None) or (None, reason)."""
|
|
if not anchor or any(c in anchor for c in "[]\n"):
|
|
return None, "anchor-not-in-sentence"
|
|
in_code = False
|
|
start = s.text.find(anchor)
|
|
while start >= 0:
|
|
end = start + len(anchor)
|
|
whole = (start == 0 or not s.text[start - 1].isalnum()) and (end == len(s.text) or not s.text[end].isalnum())
|
|
if whole:
|
|
if s.masked[start:end] == anchor:
|
|
return start, None
|
|
in_code = True
|
|
start = s.text.find(anchor, start + 1)
|
|
return None, "anchor-in-code" if in_code else "anchor-not-in-sentence"
|
|
|
|
|
|
def accept(proposals: dict[str, Any], bundle: Path, links: Path, origin: str) -> dict[str, Any]:
|
|
pages = read_pages(bundle)
|
|
sentences = {rel: {s.key: s for s in body_sentences(p.body)} for rel, p in pages.items()}
|
|
doc: dict[str, Any] = load_json(links) if links.is_file() else {
|
|
"schema": SCHEMA, "links": [], "none": [], "none_count": 0, "accepted": 0, "rejected": {}}
|
|
records: list[dict[str, Any]] = doc["links"]
|
|
rejected: Counter[str] = Counter()
|
|
accepted = 0
|
|
raw = proposals.get("links") if isinstance(proposals, dict) else None
|
|
for p in raw if isinstance(raw, list) else []:
|
|
reason, record = judge(p, pages, sentences, records)
|
|
if reason:
|
|
rejected[reason] += 1
|
|
else:
|
|
assert record is not None
|
|
records.append(record)
|
|
accepted += 1
|
|
none = proposals.get("none") if isinstance(proposals, dict) else None
|
|
nones = [str(x) for x in none] if isinstance(none, list) else []
|
|
doc["none"] += [{"id": x, "from": origin} for x in nones]
|
|
doc["none_count"] = len(doc["none"])
|
|
doc["accepted"] = len(records)
|
|
total = Counter(doc["rejected"])
|
|
total.update(rejected)
|
|
doc["rejected"] = {r: total[r] for r in REASONS if total[r]}
|
|
links.parent.mkdir(parents=True, exist_ok=True)
|
|
links.write_text(dump(doc), encoding="utf-8")
|
|
return {"accepted": accepted, "rejected": dict(rejected), "none": len(nones)}
|
|
|
|
|
|
def judge(p: Any, pages: dict[str, Page], sentences: dict[str, dict[str, Sentence]],
|
|
records: list[dict[str, Any]]) -> tuple[str | None, dict[str, Any] | None]:
|
|
if not (isinstance(p, dict) and all(isinstance(p.get(k), str) for k in ("page", "sentence", "anchor", "target"))):
|
|
return "malformed", None
|
|
rel, target, anchor, key = page_rel(p["page"]), page_rel(p["target"]), p["anchor"], p["sentence"]
|
|
s = sentences.get(rel, {}).get(key)
|
|
if s is None:
|
|
return "sentence-missing", None
|
|
if target not in pages:
|
|
return "target-missing", None
|
|
if target == rel:
|
|
return "self-link", None
|
|
if not s.footnoted:
|
|
return "no-footnote", None
|
|
at, reason = find_anchor(s, anchor)
|
|
if at is None:
|
|
return reason, None
|
|
here = [r for r in records if r["page"] == rel and r["sentence"] == key]
|
|
if any(r["target"] == target for r in here):
|
|
return "duplicate", None
|
|
if len(here) >= MAX_LINKS:
|
|
return "over-limit", None
|
|
if any(at < int(r["at"]) + len(str(r["anchor"])) and int(r["at"]) < at + len(anchor) for r in here):
|
|
return "overlap", None
|
|
return None, {"page": rel, "sentence": key, "anchor": anchor, "at": at, "target": target}
|
|
|
|
|
|
# --- command line ----------------------------------------------------------------------------------------------
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
s = sub.add_parser("strip")
|
|
s.add_argument("bundle")
|
|
s.add_argument("--out", required=True)
|
|
se = sub.add_parser("sentences")
|
|
for name in ("--accepted", "--bundle", "--out"):
|
|
se.add_argument(name, required=True)
|
|
se.add_argument("--pack", type=int, required=True)
|
|
t = sub.add_parser("task")
|
|
t.add_argument("bundle")
|
|
for name in ("--sentences", "--catalogue", "--out"):
|
|
t.add_argument(name, required=True)
|
|
a = sub.add_parser("accept")
|
|
a.add_argument("proposals")
|
|
a.add_argument("--bundle", required=True)
|
|
a.add_argument("--links", required=True)
|
|
pl = sub.add_parser("place")
|
|
pl.add_argument("bundle")
|
|
pl.add_argument("--links", required=True)
|
|
pl.add_argument("--out", required=True)
|
|
pl.add_argument("--replace", action="store_true")
|
|
b = sub.add_parser("pass-before")
|
|
for name in ("--accepted", "--bundle", "--out"):
|
|
b.add_argument(name, required=True)
|
|
b.add_argument("--catalogue")
|
|
b.add_argument("--pack", type=int, required=True)
|
|
b.add_argument("--to", type=int)
|
|
f = sub.add_parser("pass-after")
|
|
for name in ("--pass-dir", "--bundle", "--links", "--linked", "--sources", "--transcripts", "--okter", "--work",
|
|
"--kost", "--lines", "--unit"):
|
|
f.add_argument(name, required=True)
|
|
for name in ("--cap", "--done", "--total"):
|
|
f.add_argument(name, type=int, required=True)
|
|
f.add_argument("--blocks")
|
|
f.add_argument("--name")
|
|
return p
|
|
|
|
|
|
def sentences_doc(accepted: Path, pack: int, bundle: Path) -> dict[str, Any]:
|
|
wanted = pack_sentences(load_json(accepted), pack, read_pages(bundle))
|
|
return {"schema": SENTENCES_SCHEMA, "pack": pack, "pages": wanted}
|
|
|
|
|
|
def pass_before(args: argparse.Namespace) -> int:
|
|
accepted, pages = load_json(Path(args.accepted)), read_pages(Path(args.bundle))
|
|
given = read_ids(Path(args.catalogue)) if args.catalogue else None
|
|
last = args.pack if args.to is None else args.to
|
|
if last < args.pack:
|
|
raise RunError(f"--to {last} is before --pack {args.pack}")
|
|
jobs = []
|
|
for pack in range(args.pack, last + 1): # render every pack before writing any, so a failure leaves nothing behind
|
|
out = Path(args.out) if args.to is None else Path(args.out) / f"p{pack}"
|
|
wanted = pack_sentences(accepted, pack, pages)
|
|
pool = given if given is not None else pack_pool(accepted, pack, pages)
|
|
ids = filter_catalogue(accepted, pack, pool, pages)
|
|
jobs.append((pack, out, wanted, pool, ids, render_pages(pages, wanted, ids, f"pack {pack}")))
|
|
empty = 0
|
|
for pack, out, wanted, pool, ids, parts in jobs:
|
|
write_new(out / "sentences.json", dump({"schema": SENTENCES_SCHEMA, "pack": pack, "pages": wanted}))
|
|
write_parts(out, parts, {"pack": pack, "catalogue": {"concepts": len(ids), "pool": len(pool)}})
|
|
n = sum(len(v) for v in wanted.values())
|
|
empty += not n
|
|
cat = parts[0][1]["catalogue_chars"] if parts else 0
|
|
print(f"pass-before: pack {pack}, pages {len(wanted)}, sentences {n}, parts {len(parts)}, catalogue "
|
|
f"{len(ids)} of {len(pool)} concepts, {cat} characters -> {out}")
|
|
for _, stats in parts:
|
|
print(f" {out / str(stats['file'])} -> {out / ('proposals-' + str(stats['part']) + '.json')}")
|
|
return 1 if empty else 0
|
|
|
|
|
|
def counts(rejected: dict[str, int]) -> str:
|
|
return ", ".join(f"{r} {n}" for r, n in sorted(rejected.items())) or "none"
|
|
|
|
|
|
def pack_dirs(pass_dir: Path) -> list[Path]:
|
|
"""The p<n> directories of an interval, in pack order."""
|
|
dirs = [d for d in pass_dir.iterdir() if d.is_dir() and re.fullmatch(r"p\d+", d.name)]
|
|
return sorted(dirs, key=lambda d: int(d.name[1:]))
|
|
|
|
|
|
def accept_dir(directory: Path, bundle: Path, links: Path) -> tuple[int, int, Counter[str]]:
|
|
files = sorted(directory.glob("proposals-*.json"))
|
|
if not files:
|
|
raise RunError(f"no proposals-*.json in {directory}")
|
|
totals: Counter[str] = Counter()
|
|
accepted = nones = 0
|
|
for path in files:
|
|
r = accept(load_json(path), bundle, links, str(path))
|
|
accepted, nones = accepted + int(r["accepted"]), nones + int(r["none"])
|
|
totals.update(r["rejected"])
|
|
return accepted, nones, totals
|
|
|
|
|
|
def stop_blocks(kost: dict[str, Any], packs_log: dict[str, Any], lines: list[str], last: int,
|
|
done: int) -> tuple[int, list[int], int, int | None]:
|
|
"""The blocks done after a pass whose last pack is `last`: the packs up to `last` and every pack with a compiler
|
|
run in the harvest, so the measured cost and the blocks cover the same packs (order 22 session B2: the next pack's
|
|
compiler counted over the pass's blocks alone). `done` must be the blocks of the accepted packs up to `last`.
|
|
Second: the packs with a compiler run in this ledger, over whose blocks the stop line takes the next pack's share
|
|
(order 27 A3: packs built in an earlier ledger cost nothing here). Third: the blocks of the compiled packs not yet
|
|
accepted, whose compiler may run once more before the next check. Fourth: their characters (`block_chars`),
|
|
None when one of them has none (a log written before blocks had characters; motor B17)."""
|
|
blocks = {int(p["pack"]): int(p["blocks"]) for p in packs_log["packs"]}
|
|
covered = sum(int(p["blocks"]) for p in packs_log["packs"] if "accepted" in p and int(p["pack"]) <= last)
|
|
if done != covered:
|
|
raise RunError(f"--done {done} is not the blocks of the accepted packs up to pack {last} ({covered})")
|
|
compiled: set[int] = set()
|
|
for r in kost["runs"]:
|
|
if r["line"] not in lines:
|
|
continue
|
|
if r.get("pack") is None:
|
|
raise RunError(f"the run {r['description']!r} is tied to no pack: STOP")
|
|
if int(r["pack"]) not in blocks:
|
|
raise RunError(f"the run {r['description']!r} is for pack {r['pack']}, which the pack log lacks: STOP")
|
|
if r["line"] == "build":
|
|
compiled.add(int(r["pack"]))
|
|
built = {n for n in blocks if n <= last} | compiled
|
|
waiting = [p for p in packs_log["packs"] if int(p["pack"]) in built and "accepted" not in p]
|
|
pending = sum(int(p["blocks"]) for p in waiting)
|
|
chars = (sum(int(p["block_chars"]) for p in waiting)
|
|
if all(isinstance(p.get("block_chars"), int) for p in waiting) else None)
|
|
return sum(blocks[n] for n in built), sorted(compiled), pending, chars
|
|
|
|
|
|
def pass_after(args: argparse.Namespace) -> int:
|
|
pass_dir, bundle, links = Path(args.pass_dir), Path(args.bundle), Path(args.links)
|
|
dirs = pack_dirs(pass_dir)
|
|
for d in dirs: # every pack has its proposals before any is accepted, so a missing one changes nothing
|
|
if not any(d.glob("proposals-*.json")):
|
|
raise RunError(f"no proposals-*.json in {d}")
|
|
totals: Counter[str] = Counter()
|
|
accepted = nones = 0
|
|
for d in dirs or [pass_dir]:
|
|
a, n, t = accept_dir(d, bundle, links)
|
|
if dirs:
|
|
print(f"accept {d.name}: accepted {a}, rejected {sum(t.values())} ({counts(dict(t))}), none {n}")
|
|
accepted, nones = accepted + a, nones + n
|
|
totals.update(t)
|
|
print(f"accept: accepted {accepted}, rejected {sum(totals.values())} ({counts(dict(totals))}), none {nones}")
|
|
placed, stale = place(bundle, links, Path(args.linked), replace=True)
|
|
print(f"place: {placed} links placed, {stale} stale")
|
|
if stale:
|
|
return 1
|
|
report = audit.link_rule_report(audit.load_bundle(Path(args.linked), Path(args.sources)))
|
|
print(f"link rule: {report['pass']} of {report['of']}, misplaced {report['misplaced']}, broken {report['broken']}, "
|
|
f"max out-degree {report['max_out_degree']}")
|
|
harvest = ["harvest", "--transcripts", args.transcripts, "--okter", args.okter, "--work", args.work,
|
|
"--out", args.kost]
|
|
code = cost.main(harvest)
|
|
print(f"cost harvest: exit {code}")
|
|
if code:
|
|
return code
|
|
done, following, chars = args.done, None, None
|
|
if args.unit == "blocks":
|
|
if not args.blocks:
|
|
raise RunError("--unit blocks needs --blocks (the pack log)")
|
|
last = max(int(d.name[1:]) for d in dirs) if dirs else int(load_json(pass_dir / "sentences.json")["pack"])
|
|
done, packs, following, chars = stop_blocks(load_json(Path(args.kost)), load_json(Path(args.blocks)),
|
|
cost.capped_lines(args.lines), last, args.done)
|
|
print(f"done: {done} of {args.total} blocks (packs {', '.join(map(str, packs))} with a compiler run)")
|
|
stop = ["stop", "--lines", args.lines, "--cap", str(args.cap), "--unit", args.unit, "--done", str(done),
|
|
"--total", str(args.total), "--kost", args.kost]
|
|
stop += ["--blocks", args.blocks] if args.blocks else []
|
|
stop += ["--name", args.name] if args.name else []
|
|
stop += ["--next", str(following)] if following is not None else []
|
|
stop += ["--next-chars", str(chars)] if following is not None and chars is not None else []
|
|
code = cost.main(stop)
|
|
print(f"cost stop: exit {code}")
|
|
return code
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
try:
|
|
if args.command == "strip":
|
|
n = strip(Path(args.bundle), Path(args.out))
|
|
print(f"strip: {n} links reduced to their anchor text -> {args.out}")
|
|
return 0
|
|
if args.command == "sentences":
|
|
doc = sentences_doc(Path(args.accepted), args.pack, Path(args.bundle))
|
|
write_new(Path(args.out), dump(doc))
|
|
n = sum(len(v) for v in doc["pages"].values())
|
|
print(f"sentences: pack {args.pack}, {len(doc['pages'])} pages, {n} sentences -> {args.out}")
|
|
return 0 if n else 1
|
|
if args.command == "pass-before":
|
|
return pass_before(args)
|
|
if args.command == "task":
|
|
out = Path(args.out)
|
|
doc = load_json(Path(args.sentences))
|
|
label = Path(args.sentences).stem
|
|
parts = render_parts(Path(args.bundle), doc["pages"], read_ids(Path(args.catalogue)), label)
|
|
write_parts(out, parts)
|
|
n = sum(len(v) for v in doc["pages"].values())
|
|
cat = parts[0][1]["catalogue_chars"] if parts else 0
|
|
print(f"{args.command}: pages {len(doc['pages'])}, sentences {n}, parts {len(parts)}, catalogue {cat} "
|
|
f"characters -> {out}")
|
|
for _, stats in parts:
|
|
print(f" {out / str(stats['file'])} -> {out / ('proposals-' + str(stats['part']) + '.json')}")
|
|
return 0 if n else 1
|
|
if args.command == "accept":
|
|
r = accept(load_json(Path(args.proposals)), Path(args.bundle), Path(args.links), args.proposals)
|
|
print(f"accept: accepted {r['accepted']}, rejected {sum(r['rejected'].values())} "
|
|
f"({counts(r['rejected'])}), none {r['none']}")
|
|
return 0
|
|
if args.command == "place":
|
|
placed, stale = place(Path(args.bundle), Path(args.links), Path(args.out), args.replace)
|
|
print(f"place: {placed} links placed, {stale} stale -> {args.out}")
|
|
return 1 if stale else 0
|
|
return pass_after(args)
|
|
except (RunError, OSError, json.JSONDecodeError, KeyError, ValueError, TypeError) as exc:
|
|
print(f"klink cannot run: {exc}", file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|