"""K, the fragment compiler's code side (TRIAL, P1): skeleton, packs, input parts, validation, identity. No model. skeleton --kilde DIR --out FILE The concept index from the titles and level 2-3 headings of every docs page under DIR that pass concept test 2. One entry per normalised name (the first page in name order wins); an index, not pages. pack N --blocks FILE --index FILE --kilde-root DIR --work DIR [--size 5] Pack N's input for the compiler: every block of its sources with its anchor (file › section › span) and numbered lines, then the index lines whose name occurs in those blocks, then the output contract, in parts of at most PART_CHARS characters (the Read tool's cap). Prints the part paths and the output path. accept N --work DIR --blocks FILE --kilde-root DIR Validates pack N's fragments, resolves each to a concept by code (existing, merge candidate, new or rejected), updates the index and appends the accepted fragments. Prints "accepted X of N" and the rejections by reason, and how many accepted fragments resolved to a concept whose index line was not sent with the pack (the pack log keeps the ids sent; the risk of the D7 phrase filter). The work directory holds K-oppgaver/ (task parts), K-fragments/ (the compiler's JSON) and K-work/ (index.json, accepted.json, packs.json). Exit 0 = done, 2 = could not run. """ from __future__ import annotations import argparse import json import re import sys from collections import Counter from pathlib import Path from typing import Any from okfwiki.audit import sentence_spans from p1.split import docs_headings, is_meta PART_CHARS = 60_000 MERGE_JACCARD = 0.6 NEEDS_QUOTE = re.compile(r"\d|`") REASONS = ("malformed", "anchor-outside-pack", "anchor-crosses-blocks", "sentence-count", "quote-missing", "quote-not-in-span", "tests-missing", "release-note-new", "meta") class RunError(Exception): """The command cannot run as asked; nothing was written.""" def norm(name: str) -> str: text = name.replace("`", "").lower() text = re.sub(r"[^\w\s/.\-]", " ", text) words = text.split() if words and words[0] == "the": words = words[1:] words = [w[:-1] if len(w) > 3 and w.endswith("s") and not w.endswith("ss") else w for w in words] return " ".join(w.strip(".") or w for w in words) def slug(text: str) -> str: return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:60] or "x" def _unique_id(index: dict[str, Any], title: str) -> str: taken = {e["id"] for e in index["entries"]} base = slug(title) eid, k = base, 2 while eid in taken: eid, k = f"{base}-{k}", k + 1 return eid def skeleton(kilde: Path) -> dict[str, Any]: index: dict[str, Any] = {"schema": "okf-wiki/p1-index/1", "entries": []} seen: dict[str, dict[str, Any]] = {} for h in docs_headings(kilde): if is_meta(h.text): continue n = norm(h.text) if not n: continue if n in seen: if h.text not in seen[n]["aliases"]: seen[n]["aliases"].append(h.text) continue e = {"id": _unique_id(index, h.text), "title": h.text, "aliases": [h.text], "kind": "skeleton", "merge_of": None, "origin": {"file": h.file, "line": h.line}} index["entries"].append(e) seen[n] = e return index def _names(e: dict[str, Any]) -> set[str]: return {norm(a) for a in [e["title"], *e.get("aliases", [])]} - {""} def _jaccard(a: str, b: str) -> float: x, y = set(a.split()), set(b.split()) return len(x & y) / len(x | y) if x | y else 0.0 def resolve(fragment: dict[str, Any], index: dict[str, Any]) -> tuple[str, str | None]: """existing | merge-candidate | new | rejected (D6, D3b).""" by_id = {e["id"]: e for e in index["entries"]} if fragment.get("status") == "existing" and fragment.get("id") in by_id: return "existing", str(fragment["id"]) n = norm(str(fragment.get("concept", ""))) for e in index["entries"]: if n in _names(e): return "existing", str(e["id"]) if str(fragment.get("anchor", {}).get("file", "")).startswith("release-"): return "rejected", "release-note-new" if is_meta(str(fragment.get("concept", ""))): return "rejected", "meta" best = max(((max(_jaccard(n, m) for m in _names(e)), str(e["id"])) for e in index["entries"] if _names(e)), default=(0.0, "")) if best[0] >= MERGE_JACCARD: return "merge-candidate", best[1] return "new", None def relevant_lines(index: dict[str, Any], text: str) -> list[dict[str, Any]]: """Index entries whose normalised name or alias occurs as a phrase in the normalised text (D7).""" hay = f" {norm(text)} " return [e for e in index["entries"] if any(f" {m} " in hay for m in _names(e))] def packs(names: list[str], size: int = 5) -> list[list[str]]: return [names[i:i + size] for i in range(0, len(names), size)] CONTRACT = """## Output contract Write one JSON object `{"fragments": [...]}` to the output path named in your prompt. Each fragment: `concept`, `status` ("existing" with the index `id`, or "new" with `tests`: {"topic_shape", "citation"}), `text` (1-3 sentences), `anchor` {"file", "start", "end"} inside one block above, and `quote` (one piece copied from one line inside the anchor; required when the text holds a digit or a backticked name). A new concept only from a docs page, never from a release note. """ def render_input(pack_no: int, names: list[str], blocks_doc: dict[str, Any], index_lines: list[dict[str, Any]], kilde: Path) -> list[str]: lines = {n: (kilde / n).read_text(encoding="utf-8").split("\n") for n in names} pieces: list[str] = [f"# K pack {pack_no}: {len(names)} sources\n\nSources in order: {', '.join(names)}\n\n" + CONTRACT + "\n## Blocks\n\n"] for b in blocks_doc["blocks"]: if b["file"] not in names: continue head = f"### {b['file']} › {b['section'] or '-'} › {b['start']}–{b['end']}" if b.get("version"): head += f" (version {b['version']})" body = [f"{n}: {lines[b['file']][n - 1]}" for n in range(b["start"], b["end"] + 1)] chunk = head + "\n" for row in body: if len(chunk) + len(row) + 1 > PART_CHARS // 2: pieces.append(chunk + "\n") chunk = head + " (continued)\n" chunk += row + "\n" pieces.append(chunk + "\n") pieces.append("## Index (concepts named in these blocks)\n\n") for e in index_lines: aliases = "; ".join(a for a in e["aliases"] if a != e["title"]) pieces.append(f"- id: {e['id']} | {e['title']}" + (f" | aliases: {aliases}" if aliases else "") + "\n") room = PART_CHARS - len("\n") parts: list[str] = [] current = "" for piece in pieces: if current and len(current) + len(piece) > room: parts.append(current) current = "" current += piece if current: parts.append(current) total = len(parts) return [f"\n" + p for i, p in enumerate(parts, 1)] def validate(fragments: list[Any], names: list[str], blocks_doc: dict[str, Any], kilde: Path ) -> tuple[list[dict[str, Any]], Counter[str]]: by_file: dict[str, list[dict[str, Any]]] = {} for b in blocks_doc["blocks"]: by_file.setdefault(b["file"], []).append(b) texts: dict[str, list[str]] = {} accepted: list[dict[str, Any]] = [] rejected: Counter[str] = Counter() for f in fragments: a = f.get("anchor") if isinstance(f, dict) else None if not (isinstance(f, dict) and isinstance(f.get("concept"), str) and f["concept"].strip() and f.get("status") in ("new", "existing") and isinstance(f.get("text"), str) and isinstance(a, dict) and isinstance(a.get("file"), str) and type(a.get("start")) is int and type(a.get("end")) is int and a["start"] <= a["end"]): rejected["malformed"] += 1 continue if a["file"] not in names: rejected["anchor-outside-pack"] += 1 continue block = next((b for b in by_file.get(a["file"], []) if b["start"] <= a["start"] and a["end"] <= b["end"]), None) if block is None: rejected["anchor-crosses-blocks"] += 1 continue if not 1 <= len(sentence_spans(f["text"].strip())) <= 3: rejected["sentence-count"] += 1 continue if a["file"] not in texts: texts[a["file"]] = (kilde / a["file"]).read_text(encoding="utf-8").split("\n") span = texts[a["file"]][a["start"] - 1:a["end"]] quote = f.get("quote") if not (isinstance(quote, str) and quote.strip()): if NEEDS_QUOTE.search(f["text"]): rejected["quote-missing"] += 1 continue quote, quote_line = None, None else: hit = next((a["start"] + i for i, line in enumerate(span) if quote in line), None) if hit is None: rejected["quote-not-in-span"] += 1 continue quote_line = hit tests = f.get("tests") if f["status"] == "new" and not (isinstance(tests, dict) and str(tests.get("topic_shape", "")).strip() and str(tests.get("citation", "")).strip()): rejected["tests-missing"] += 1 continue accepted.append({**f, "quote": quote, "quote_line": quote_line, "version": block.get("version"), "section": block["section"], "position": names.index(a["file"]) + 1}) return accepted, rejected def _read_json(path: Path) -> Any: try: return json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise RunError(f"cannot read {path}: {exc}") from exc def _write_json(path: Path, data: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") def _pack_log(work: Path) -> dict[str, Any]: path = work / "K-work" / "packs.json" return _read_json(path) if path.is_file() else {"schema": "okf-wiki/p1-packs/1", "packs": []} def _record(log: dict[str, Any], n: int) -> dict[str, Any]: for r in log["packs"]: if r["pack"] == n: return r # type: ignore[no-any-return] r = {"pack": n} log["packs"].append(r) return r def cmd_pack(args: argparse.Namespace) -> int: work, n = Path(args.work), args.n doc = _read_json(Path(args.blocks)) all_packs = packs([str(x) for x in doc["order"]], args.size) if not 1 <= n <= len(all_packs): raise RunError(f"pack {n} does not exist; there are {len(all_packs)}") names = all_packs[n - 1] kilde = Path(args.kilde_root) / "kilde" index = _read_json(Path(args.index)) text = "\n".join((kilde / name).read_text(encoding="utf-8") for name in names) lines = relevant_lines(index, text) parts = render_input(n, names, doc, lines, kilde) out_dir = work / "K-oppgaver" paths = [out_dir / f"pack-{n}-part-{i}.md" for i in range(1, len(parts) + 1)] if any(p.exists() for p in paths): raise RunError(f"pack {n}'s task parts exist in {out_dir}; refusing to overwrite") out_dir.mkdir(parents=True, exist_ok=True) for p, text_part in zip(paths, parts): p.write_text(text_part, encoding="utf-8") log = _pack_log(work) rec = _record(log, n) rec.update(sources=names, blocks=sum(b["file"] in names for b in doc["blocks"]), index_lines=len(lines), index_total=len(index["entries"]), index_ids=[str(e["id"]) for e in lines], parts=len(parts), chars=sum(len(p) for p in parts)) _write_json(work / "K-work" / "packs.json", log) for p in paths: print(f"part: {p}") print(f"output: {work / 'K-fragments' / f'pack-{n}.json'}") print(f"index lines sent: {len(lines)} of {len(index['entries'])}") return 0 def cmd_accept(args: argparse.Namespace) -> int: work, n = Path(args.work), args.n doc = _read_json(Path(args.blocks)) names = packs([str(x) for x in doc["order"]], args.size)[n - 1] kilde = Path(args.kilde_root) / "kilde" raw = _read_json(work / "K-fragments" / f"pack-{n}.json") fragments = raw.get("fragments") if isinstance(raw, dict) else None if not isinstance(fragments, list): raise RunError(f"pack {n}'s output has no fragments list") index_path = work / "K-work" / "index.json" index = _read_json(index_path) valid, rejected = validate(fragments, names, doc, kilde) acc_path = work / "K-work" / "accepted.json" acc: dict[str, Any] = (_read_json(acc_path) if acc_path.is_file() else {"schema": "okf-wiki/p1-accepted/1", "fragments": []}) if any(a["pack"] == n for a in acc["fragments"]): raise RunError(f"pack {n} is already accepted") sent = set(_record(_pack_log(work), n).get("index_ids", [])) not_sent: Counter[str] = Counter() accepted = 0 for f in valid: outcome, ref = resolve(f, index) if outcome == "rejected": rejected[str(ref)] += 1 continue if outcome == "existing": cid = str(ref) else: cid = _unique_id(index, f["concept"]) index["entries"].append({"id": cid, "title": f["concept"], "aliases": [f["concept"]], "kind": outcome, "merge_of": ref, "origin": {"file": f["anchor"]["file"], "line": f["anchor"]["start"]}, "tests": f.get("tests")}) acc["fragments"].append({"pack": n, "position": f["position"], "file": f["anchor"]["file"], "start": f["anchor"]["start"], "end": f["anchor"]["end"], "section": f["section"], "text": f["text"].strip(), "quote": f["quote"], "quote_line": f["quote_line"], "version": f["version"], "concept_id": cid, "outcome": outcome}) if cid not in sent: not_sent[outcome] += 1 accepted += 1 _write_json(index_path, index) _write_json(acc_path, acc) log = _pack_log(work) rec = _record(log, n) rec.update(proposed=len(fragments), accepted=accepted, rejected=dict(sorted(rejected.items())), not_sent={"total": sum(not_sent.values()), "by_outcome": dict(sorted(not_sent.items()))}) _write_json(work / "K-work" / "packs.json", log) print(f"pack {n}: accepted {accepted} of {len(fragments)}") print(f" accepted with a concept not among the index lines sent: {sum(not_sent.values())}") for reason, count in sorted(rejected.items()): print(f" rejected {reason}: {count}") return 0 def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) s = sub.add_parser("skeleton") s.add_argument("--kilde", required=True) s.add_argument("--out", required=True) for name in ("pack", "accept"): c = sub.add_parser(name) c.add_argument("n", type=int) c.add_argument("--blocks", required=True) c.add_argument("--kilde-root", required=True) c.add_argument("--work", required=True) c.add_argument("--size", type=int, default=5) if name == "pack": c.add_argument("--index", required=True) return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: if args.command == "skeleton": out = Path(args.out) if out.exists(): raise RunError(f"{out} exists; refusing to overwrite") index = skeleton(Path(args.kilde)) _write_json(out, index) print(f"skeleton: {len(index['entries'])} entries -> {out}") return 0 if args.command == "pack": return cmd_pack(args) return cmd_accept(args) except (RunError, OSError) as exc: print(f"kpack cannot run: {exc}", file=sys.stderr) return 2 if __name__ == "__main__": sys.exit(main())