First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
374 lines
16 KiB
Python
374 lines
16 KiB
Python
"""K, the fragment compiler's code side (TRIAL, P1): skeleton, packs, input parts, validation, identity. No model.
|
||
|
||
skeleton --kilde DIR --out FILE
|
||
The concept index from the titles and level 2-3 headings of every docs page under DIR that pass concept
|
||
test 2. One entry per normalised name (the first page in name order wins); an index, not pages.
|
||
pack N --blocks FILE --index FILE --kilde-root DIR --work DIR [--size 5]
|
||
Pack N's input for the compiler: every block of its sources with its anchor (file › section › span) and
|
||
numbered lines, then the index lines whose name occurs in those blocks, then the output contract, in parts
|
||
of at most PART_CHARS characters (the Read tool's cap). Prints the part paths and the output path.
|
||
accept N --work DIR --blocks FILE --kilde-root DIR
|
||
Validates pack N's fragments, resolves each to a concept by code (existing, merge candidate, new or
|
||
rejected), updates the index and appends the accepted fragments. Prints "accepted X of N" and the
|
||
rejections by reason, and how many accepted fragments resolved to a concept whose index line was not
|
||
sent with the pack (the pack log keeps the ids sent; the risk of the D7 phrase filter).
|
||
|
||
The work directory holds K-oppgaver/ (task parts), K-fragments/ (the compiler's JSON) and K-work/ (index.json,
|
||
accepted.json, packs.json). Exit 0 = done, 2 = could not run.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import sys
|
||
from collections import Counter
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from okfwiki.audit import sentence_spans
|
||
from p1.split import docs_headings, is_meta
|
||
|
||
PART_CHARS = 60_000
|
||
MERGE_JACCARD = 0.6
|
||
NEEDS_QUOTE = re.compile(r"\d|`")
|
||
REASONS = ("malformed", "anchor-outside-pack", "anchor-crosses-blocks", "sentence-count", "quote-missing",
|
||
"quote-not-in-span", "tests-missing", "release-note-new", "meta")
|
||
|
||
|
||
class RunError(Exception):
|
||
"""The command cannot run as asked; nothing was written."""
|
||
|
||
|
||
def norm(name: str) -> str:
|
||
text = name.replace("`", "").lower()
|
||
text = re.sub(r"[^\w\s/.\-]", " ", text)
|
||
words = text.split()
|
||
if words and words[0] == "the":
|
||
words = words[1:]
|
||
words = [w[:-1] if len(w) > 3 and w.endswith("s") and not w.endswith("ss") else w for w in words]
|
||
return " ".join(w.strip(".") or w for w in words)
|
||
|
||
|
||
def slug(text: str) -> str:
|
||
return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:60] or "x"
|
||
|
||
|
||
def _unique_id(index: dict[str, Any], title: str) -> str:
|
||
taken = {e["id"] for e in index["entries"]}
|
||
base = slug(title)
|
||
eid, k = base, 2
|
||
while eid in taken:
|
||
eid, k = f"{base}-{k}", k + 1
|
||
return eid
|
||
|
||
|
||
def skeleton(kilde: Path) -> dict[str, Any]:
|
||
index: dict[str, Any] = {"schema": "okf-wiki/p1-index/1", "entries": []}
|
||
seen: dict[str, dict[str, Any]] = {}
|
||
for h in docs_headings(kilde):
|
||
if is_meta(h.text):
|
||
continue
|
||
n = norm(h.text)
|
||
if not n:
|
||
continue
|
||
if n in seen:
|
||
if h.text not in seen[n]["aliases"]:
|
||
seen[n]["aliases"].append(h.text)
|
||
continue
|
||
e = {"id": _unique_id(index, h.text), "title": h.text, "aliases": [h.text], "kind": "skeleton",
|
||
"merge_of": None, "origin": {"file": h.file, "line": h.line}}
|
||
index["entries"].append(e)
|
||
seen[n] = e
|
||
return index
|
||
|
||
|
||
def _names(e: dict[str, Any]) -> set[str]:
|
||
return {norm(a) for a in [e["title"], *e.get("aliases", [])]} - {""}
|
||
|
||
|
||
def _jaccard(a: str, b: str) -> float:
|
||
x, y = set(a.split()), set(b.split())
|
||
return len(x & y) / len(x | y) if x | y else 0.0
|
||
|
||
|
||
def resolve(fragment: dict[str, Any], index: dict[str, Any]) -> tuple[str, str | None]:
|
||
"""existing <id> | merge-candidate <id> | new | rejected <reason> (D6, D3b)."""
|
||
by_id = {e["id"]: e for e in index["entries"]}
|
||
if fragment.get("status") == "existing" and fragment.get("id") in by_id:
|
||
return "existing", str(fragment["id"])
|
||
n = norm(str(fragment.get("concept", "")))
|
||
for e in index["entries"]:
|
||
if n in _names(e):
|
||
return "existing", str(e["id"])
|
||
if str(fragment.get("anchor", {}).get("file", "")).startswith("release-"):
|
||
return "rejected", "release-note-new"
|
||
if is_meta(str(fragment.get("concept", ""))):
|
||
return "rejected", "meta"
|
||
best = max(((max(_jaccard(n, m) for m in _names(e)), str(e["id"])) for e in index["entries"] if _names(e)),
|
||
default=(0.0, ""))
|
||
if best[0] >= MERGE_JACCARD:
|
||
return "merge-candidate", best[1]
|
||
return "new", None
|
||
|
||
|
||
def relevant_lines(index: dict[str, Any], text: str) -> list[dict[str, Any]]:
|
||
"""Index entries whose normalised name or alias occurs as a phrase in the normalised text (D7)."""
|
||
hay = f" {norm(text)} "
|
||
return [e for e in index["entries"] if any(f" {m} " in hay for m in _names(e))]
|
||
|
||
|
||
def packs(names: list[str], size: int = 5) -> list[list[str]]:
|
||
return [names[i:i + size] for i in range(0, len(names), size)]
|
||
|
||
|
||
CONTRACT = """## Output contract
|
||
|
||
Write one JSON object `{"fragments": [...]}` to the output path named in your prompt. Each fragment:
|
||
`concept`, `status` ("existing" with the index `id`, or "new" with `tests`: {"topic_shape", "citation"}),
|
||
`text` (1-3 sentences), `anchor` {"file", "start", "end"} inside one block above, and `quote` (one piece copied from
|
||
one line inside the anchor; required when the text holds a digit or a backticked name). A new concept only from a docs
|
||
page, never from a release note.
|
||
"""
|
||
|
||
|
||
def render_input(pack_no: int, names: list[str], blocks_doc: dict[str, Any], index_lines: list[dict[str, Any]],
|
||
kilde: Path) -> list[str]:
|
||
lines = {n: (kilde / n).read_text(encoding="utf-8").split("\n") for n in names}
|
||
pieces: list[str] = [f"# K pack {pack_no}: {len(names)} sources\n\nSources in order: {', '.join(names)}\n\n"
|
||
+ CONTRACT + "\n## Blocks\n\n"]
|
||
for b in blocks_doc["blocks"]:
|
||
if b["file"] not in names:
|
||
continue
|
||
head = f"### {b['file']} › {b['section'] or '-'} › {b['start']}–{b['end']}"
|
||
if b.get("version"):
|
||
head += f" (version {b['version']})"
|
||
body = [f"{n}: {lines[b['file']][n - 1]}" for n in range(b["start"], b["end"] + 1)]
|
||
chunk = head + "\n"
|
||
for row in body:
|
||
if len(chunk) + len(row) + 1 > PART_CHARS // 2:
|
||
pieces.append(chunk + "\n")
|
||
chunk = head + " (continued)\n"
|
||
chunk += row + "\n"
|
||
pieces.append(chunk + "\n")
|
||
pieces.append("## Index (concepts named in these blocks)\n\n")
|
||
for e in index_lines:
|
||
aliases = "; ".join(a for a in e["aliases"] if a != e["title"])
|
||
pieces.append(f"- id: {e['id']} | {e['title']}" + (f" | aliases: {aliases}" if aliases else "") + "\n")
|
||
room = PART_CHARS - len("<!-- part 99 of 99 -->\n")
|
||
parts: list[str] = []
|
||
current = ""
|
||
for piece in pieces:
|
||
if current and len(current) + len(piece) > room:
|
||
parts.append(current)
|
||
current = ""
|
||
current += piece
|
||
if current:
|
||
parts.append(current)
|
||
total = len(parts)
|
||
return [f"<!-- part {i} of {total} -->\n" + p for i, p in enumerate(parts, 1)]
|
||
|
||
|
||
def validate(fragments: list[Any], names: list[str], blocks_doc: dict[str, Any], kilde: Path
|
||
) -> tuple[list[dict[str, Any]], Counter[str]]:
|
||
by_file: dict[str, list[dict[str, Any]]] = {}
|
||
for b in blocks_doc["blocks"]:
|
||
by_file.setdefault(b["file"], []).append(b)
|
||
texts: dict[str, list[str]] = {}
|
||
accepted: list[dict[str, Any]] = []
|
||
rejected: Counter[str] = Counter()
|
||
for f in fragments:
|
||
a = f.get("anchor") if isinstance(f, dict) else None
|
||
if not (isinstance(f, dict) and isinstance(f.get("concept"), str) and f["concept"].strip()
|
||
and f.get("status") in ("new", "existing") and isinstance(f.get("text"), str)
|
||
and isinstance(a, dict) and isinstance(a.get("file"), str)
|
||
and type(a.get("start")) is int and type(a.get("end")) is int and a["start"] <= a["end"]):
|
||
rejected["malformed"] += 1
|
||
continue
|
||
if a["file"] not in names:
|
||
rejected["anchor-outside-pack"] += 1
|
||
continue
|
||
block = next((b for b in by_file.get(a["file"], []) if b["start"] <= a["start"] and a["end"] <= b["end"]), None)
|
||
if block is None:
|
||
rejected["anchor-crosses-blocks"] += 1
|
||
continue
|
||
if not 1 <= len(sentence_spans(f["text"].strip())) <= 3:
|
||
rejected["sentence-count"] += 1
|
||
continue
|
||
if a["file"] not in texts:
|
||
texts[a["file"]] = (kilde / a["file"]).read_text(encoding="utf-8").split("\n")
|
||
span = texts[a["file"]][a["start"] - 1:a["end"]]
|
||
quote = f.get("quote")
|
||
if not (isinstance(quote, str) and quote.strip()):
|
||
if NEEDS_QUOTE.search(f["text"]):
|
||
rejected["quote-missing"] += 1
|
||
continue
|
||
quote, quote_line = None, None
|
||
else:
|
||
hit = next((a["start"] + i for i, line in enumerate(span) if quote in line), None)
|
||
if hit is None:
|
||
rejected["quote-not-in-span"] += 1
|
||
continue
|
||
quote_line = hit
|
||
tests = f.get("tests")
|
||
if f["status"] == "new" and not (isinstance(tests, dict) and str(tests.get("topic_shape", "")).strip()
|
||
and str(tests.get("citation", "")).strip()):
|
||
rejected["tests-missing"] += 1
|
||
continue
|
||
accepted.append({**f, "quote": quote, "quote_line": quote_line, "version": block.get("version"),
|
||
"section": block["section"], "position": names.index(a["file"]) + 1})
|
||
return accepted, rejected
|
||
|
||
|
||
def _read_json(path: Path) -> Any:
|
||
try:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
except (OSError, json.JSONDecodeError) as exc:
|
||
raise RunError(f"cannot read {path}: {exc}") from exc
|
||
|
||
|
||
def _write_json(path: Path, data: Any) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(json.dumps(data, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
||
|
||
|
||
def _pack_log(work: Path) -> dict[str, Any]:
|
||
path = work / "K-work" / "packs.json"
|
||
return _read_json(path) if path.is_file() else {"schema": "okf-wiki/p1-packs/1", "packs": []}
|
||
|
||
|
||
def _record(log: dict[str, Any], n: int) -> dict[str, Any]:
|
||
for r in log["packs"]:
|
||
if r["pack"] == n:
|
||
return r # type: ignore[no-any-return]
|
||
r = {"pack": n}
|
||
log["packs"].append(r)
|
||
return r
|
||
|
||
|
||
def cmd_pack(args: argparse.Namespace) -> int:
|
||
work, n = Path(args.work), args.n
|
||
doc = _read_json(Path(args.blocks))
|
||
all_packs = packs([str(x) for x in doc["order"]], args.size)
|
||
if not 1 <= n <= len(all_packs):
|
||
raise RunError(f"pack {n} does not exist; there are {len(all_packs)}")
|
||
names = all_packs[n - 1]
|
||
kilde = Path(args.kilde_root) / "kilde"
|
||
index = _read_json(Path(args.index))
|
||
text = "\n".join((kilde / name).read_text(encoding="utf-8") for name in names)
|
||
lines = relevant_lines(index, text)
|
||
parts = render_input(n, names, doc, lines, kilde)
|
||
out_dir = work / "K-oppgaver"
|
||
paths = [out_dir / f"pack-{n}-part-{i}.md" for i in range(1, len(parts) + 1)]
|
||
if any(p.exists() for p in paths):
|
||
raise RunError(f"pack {n}'s task parts exist in {out_dir}; refusing to overwrite")
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
for p, text_part in zip(paths, parts):
|
||
p.write_text(text_part, encoding="utf-8")
|
||
log = _pack_log(work)
|
||
rec = _record(log, n)
|
||
rec.update(sources=names, blocks=sum(b["file"] in names for b in doc["blocks"]), index_lines=len(lines),
|
||
index_total=len(index["entries"]), index_ids=[str(e["id"]) for e in lines], parts=len(parts),
|
||
chars=sum(len(p) for p in parts))
|
||
_write_json(work / "K-work" / "packs.json", log)
|
||
for p in paths:
|
||
print(f"part: {p}")
|
||
print(f"output: {work / 'K-fragments' / f'pack-{n}.json'}")
|
||
print(f"index lines sent: {len(lines)} of {len(index['entries'])}")
|
||
return 0
|
||
|
||
|
||
def cmd_accept(args: argparse.Namespace) -> int:
|
||
work, n = Path(args.work), args.n
|
||
doc = _read_json(Path(args.blocks))
|
||
names = packs([str(x) for x in doc["order"]], args.size)[n - 1]
|
||
kilde = Path(args.kilde_root) / "kilde"
|
||
raw = _read_json(work / "K-fragments" / f"pack-{n}.json")
|
||
fragments = raw.get("fragments") if isinstance(raw, dict) else None
|
||
if not isinstance(fragments, list):
|
||
raise RunError(f"pack {n}'s output has no fragments list")
|
||
index_path = work / "K-work" / "index.json"
|
||
index = _read_json(index_path)
|
||
valid, rejected = validate(fragments, names, doc, kilde)
|
||
acc_path = work / "K-work" / "accepted.json"
|
||
acc: dict[str, Any] = (_read_json(acc_path) if acc_path.is_file()
|
||
else {"schema": "okf-wiki/p1-accepted/1", "fragments": []})
|
||
if any(a["pack"] == n for a in acc["fragments"]):
|
||
raise RunError(f"pack {n} is already accepted")
|
||
sent = set(_record(_pack_log(work), n).get("index_ids", []))
|
||
not_sent: Counter[str] = Counter()
|
||
accepted = 0
|
||
for f in valid:
|
||
outcome, ref = resolve(f, index)
|
||
if outcome == "rejected":
|
||
rejected[str(ref)] += 1
|
||
continue
|
||
if outcome == "existing":
|
||
cid = str(ref)
|
||
else:
|
||
cid = _unique_id(index, f["concept"])
|
||
index["entries"].append({"id": cid, "title": f["concept"], "aliases": [f["concept"]], "kind": outcome,
|
||
"merge_of": ref, "origin": {"file": f["anchor"]["file"], "line": f["anchor"]["start"]},
|
||
"tests": f.get("tests")})
|
||
acc["fragments"].append({"pack": n, "position": f["position"], "file": f["anchor"]["file"],
|
||
"start": f["anchor"]["start"], "end": f["anchor"]["end"], "section": f["section"],
|
||
"text": f["text"].strip(), "quote": f["quote"], "quote_line": f["quote_line"],
|
||
"version": f["version"], "concept_id": cid, "outcome": outcome})
|
||
if cid not in sent:
|
||
not_sent[outcome] += 1
|
||
accepted += 1
|
||
_write_json(index_path, index)
|
||
_write_json(acc_path, acc)
|
||
log = _pack_log(work)
|
||
rec = _record(log, n)
|
||
rec.update(proposed=len(fragments), accepted=accepted, rejected=dict(sorted(rejected.items())),
|
||
not_sent={"total": sum(not_sent.values()), "by_outcome": dict(sorted(not_sent.items()))})
|
||
_write_json(work / "K-work" / "packs.json", log)
|
||
print(f"pack {n}: accepted {accepted} of {len(fragments)}")
|
||
print(f" accepted with a concept not among the index lines sent: {sum(not_sent.values())}")
|
||
for reason, count in sorted(rejected.items()):
|
||
print(f" rejected {reason}: {count}")
|
||
return 0
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
sub = p.add_subparsers(dest="command", required=True)
|
||
s = sub.add_parser("skeleton")
|
||
s.add_argument("--kilde", required=True)
|
||
s.add_argument("--out", required=True)
|
||
for name in ("pack", "accept"):
|
||
c = sub.add_parser(name)
|
||
c.add_argument("n", type=int)
|
||
c.add_argument("--blocks", required=True)
|
||
c.add_argument("--kilde-root", required=True)
|
||
c.add_argument("--work", required=True)
|
||
c.add_argument("--size", type=int, default=5)
|
||
if name == "pack":
|
||
c.add_argument("--index", required=True)
|
||
return p
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
args = build_parser().parse_args(argv)
|
||
try:
|
||
if args.command == "skeleton":
|
||
out = Path(args.out)
|
||
if out.exists():
|
||
raise RunError(f"{out} exists; refusing to overwrite")
|
||
index = skeleton(Path(args.kilde))
|
||
_write_json(out, index)
|
||
print(f"skeleton: {len(index['entries'])} entries -> {out}")
|
||
return 0
|
||
if args.command == "pack":
|
||
return cmd_pack(args)
|
||
return cmd_accept(args)
|
||
except (RunError, OSError) as exc:
|
||
print(f"kpack cannot run: {exc}", file=sys.stderr)
|
||
return 2
|
||
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|