First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
540 lines
25 KiB
Python
540 lines
25 KiB
Python
"""The direction probe R (PLAN.md § Leveranser, row R): three arms answer the same questions from the same sources.
|
|
|
|
filsok file search alone over the sources in the cut
|
|
wiki the okf-wiki plus file search, following the skill's router (index.md, the pages, the raw source)
|
|
astro a wiki built with Astro-Han's karpathy-llm-wiki skill on the same sources, following its Query steps
|
|
|
|
Each question is answered by one isolated subagent per arm in the user's session. This module never calls a model:
|
|
|
|
fetch-astro the skill's Fetch step done mechanically: each source copied byte-exact under the raw template's
|
|
header into <project>/raw/<topic>/<name> (the sources are local, clean Markdown already)
|
|
setup one arm's basis (a copy, so no arm reads another's folder) and one task file per question, which
|
|
carries the question and never the answer key, the evidence or the count
|
|
collect the per-question answer files into the answers files the gates read
|
|
harvest tokens (every usage field, each message id once) and isolation breaches from the transcripts
|
|
score the gates' per-question results for the probe questions only: per arm and group, and in pairs
|
|
|
|
The gates are run unchanged (`gates/multidoc.py gate`, `gates/wholedoc.py gate`, each with --out).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import sys
|
|
from itertools import combinations
|
|
from math import comb
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
ARMS = ("filsok", "wiki", "astro")
|
|
GROUPS = ("flerdokument", "helt dokument", "telling")
|
|
MD_SCHEMA = "okf-wiki/multidoc-set/1"
|
|
WD_SCHEMA = "okf-wiki/wholedoc-set/1"
|
|
ALLOWED_TOOLS = ("Read", "Grep", "Glob", "Bash", "Write", "SubagentHandback")
|
|
READ_ONLY = ("grep", "egrep", "fgrep", "rg", "find", "ls", "wc", "head", "tail", "sed", "cat", "sort", "uniq", "cut",
|
|
"tr", "nl")
|
|
SEPARATORS = ("|", "&&", ";", "||")
|
|
FIND_ACTIONS = ("-exec", "-execdir", "-ok", "-okdir", "-delete", "-fprint", "-fprint0", "-fprintf", "-fls")
|
|
USAGE_FIELDS = ("input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens", "output_tokens")
|
|
|
|
|
|
class ProbeError(Exception):
|
|
"""The command cannot run; reported as FAIL with exit 1."""
|
|
|
|
|
|
def load_json(path: Path) -> Any:
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise ProbeError(f"cannot read {path}: {exc}") from exc
|
|
|
|
|
|
def write_json(path: Path, data: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
|
|
|
|
|
def copy_names(sources: Path, names: list[str], target: Path) -> None:
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
for name in names:
|
|
src = sources / name
|
|
if not src.is_file():
|
|
raise ProbeError(f"source {src} is missing")
|
|
shutil.copyfile(src, target / name)
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- fetch-astro
|
|
|
|
|
|
def fetch_astro(sources: Path, names: list[str], project: Path, topic: str, collected: str) -> list[Path]:
|
|
"""The Fetch step of karpathy-llm-wiki, per its raw template, with the source text preserved byte-exact."""
|
|
folder = project / "raw" / topic
|
|
targets = [folder / name for name in names]
|
|
existing = [str(t) for t in targets if t.exists()]
|
|
if existing:
|
|
raise ProbeError(f"raw files exist already and are never overwritten: {', '.join(existing)}")
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
for name, target in zip(names, targets):
|
|
src = sources / name
|
|
if not src.is_file():
|
|
raise ProbeError(f"source {src} is missing")
|
|
header = (f"# {Path(name).stem}\n\n> Source: kilde/{name}\n> Collected: {collected}\n"
|
|
"> Published: Unknown\n\n")
|
|
target.write_bytes(header.encode("utf-8") + src.read_bytes())
|
|
return targets
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- setup
|
|
|
|
|
|
SEARCH = {
|
|
"filsok": """BASIS holds the source files and nothing else. Search them with grep and find (Bash, as above), and read
|
|
what you find with Read.""",
|
|
"wiki": """BASIS holds `wiki/`, a knowledge wiki compiled from the sources, and `kilde/`, the source files. A page
|
|
names its source as `/kilde/<name>`, which is the file `BASIS/kilde/<name>`. Follow this order:
|
|
1. Read `BASIS/wiki/index.md` first.
|
|
2. Read the pages the index points to.
|
|
3. Open the raw source behind a footnote (`resource` at `x-locator`) when the quote has to be checked or when the
|
|
page does not say enough.
|
|
You may also search `wiki/` and `kilde/` with grep and find (Bash, as above) at any point.""",
|
|
"astro": """BASIS holds `wiki/`, a knowledge wiki compiled from the sources with the karpathy-llm-wiki skill, and
|
|
`raw/`, the source files it was compiled from (each raw file starts with a short metadata header; the source text
|
|
follows it unchanged). Follow the skill's Query steps:
|
|
1. Read `BASIS/wiki/index.md` to locate candidate articles, then full-text search `wiki/` with the topic's key terms
|
|
and their synonyms (grep through Bash, as above).
|
|
2. Read the articles you found and synthesize an answer.
|
|
3. Prefer wiki content over your own training knowledge.
|
|
Each article links the raw files it was compiled from in its `Raw` field. Open those raw files when you need an exact
|
|
value or a verbatim quote.""",
|
|
}
|
|
|
|
CITE_NAME = {
|
|
"filsok": "the bare name of a source file in BASIS",
|
|
"wiki": "the bare name of a source file in `BASIS/kilde`",
|
|
"astro": "the bare name of a raw file in `BASIS/raw/<topic>/`",
|
|
}
|
|
|
|
|
|
def render_task(arm: str, basis: Path, qid: str, question: str, is_count: bool, answer_path: Path) -> str:
|
|
count_line = '"count": <integer>, ' if is_count else ""
|
|
count_rule = ("\n- `count`: the number the question asks for, as a JSON integer. Say in `answer` how you counted."
|
|
if is_count else "")
|
|
return f"""# Answer one question
|
|
|
|
BASIS: `{basis}`
|
|
|
|
You answer ONE question from the files in BASIS. Read nothing outside BASIS apart from this task file: no other
|
|
folder, no web, no earlier work. Answer only from what the files say, and say plainly when they do not say something.
|
|
|
|
Tools: Read (a file inside BASIS), Bash for read-only search, and Write once, for the answer file. No other tool.
|
|
Every Bash command starts with `cd {basis} && ` and then uses only {', '.join(READ_ONLY)}, joined by `|` or `&&`,
|
|
with paths relative to BASIS: never `..`, `~` or a path outside BASIS, no `$` expansion, no output redirection except
|
|
`2>/dev/null`, no `find -exec` or `-delete`, no `sed -i`. For example: `cd {basis} && grep -rn "term" .`
|
|
|
|
## How to search
|
|
|
|
{SEARCH[arm]}
|
|
|
|
## The question
|
|
|
|
{question}
|
|
|
|
## The answer
|
|
|
|
Write one JSON file to `{answer_path}`:
|
|
|
|
```json
|
|
{{"id": "{qid}", "answer": "<your answer in prose>", {count_line}"citations": [{{"file": "<name>", "quote": "<text>"}}]}}
|
|
```
|
|
|
|
- `answer`: answer every part of the question.{count_rule}
|
|
- `citations`: one or more per source file your answer relies on, and cite every such file.
|
|
- `file`: {CITE_NAME[arm]}, for example `notes.md`: never a path and never a wiki page.
|
|
- `quote`: text copied character for character from that file (only whitespace may differ), long enough to carry
|
|
the fact, for example the whole line or sentence.
|
|
|
|
When the file is written, reply with the one word `done`.
|
|
"""
|
|
|
|
|
|
def probe_questions(md_set: Path, md_ids: list[str], wd_set: Path) -> list[tuple[str, dict[str, Any]]]:
|
|
md, wd = load_json(md_set), load_json(wd_set)
|
|
if not isinstance(md, dict) or md.get("schema") != MD_SCHEMA:
|
|
raise ProbeError(f"{md_set} is not a set with schema {MD_SCHEMA}")
|
|
if not isinstance(wd, dict) or wd.get("schema") != WD_SCHEMA:
|
|
raise ProbeError(f"{wd_set} is not a set with schema {WD_SCHEMA}")
|
|
by_id = {q["id"]: q for q in md["questions"]}
|
|
missing = [i for i in md_ids if i not in by_id]
|
|
if missing:
|
|
raise ProbeError(f"probe ids missing from {md_set}: {', '.join(missing)}")
|
|
return [("md", by_id[i]) for i in md_ids] + [("wd", q) for q in wd["questions"]]
|
|
|
|
|
|
def setup_arm(run: Path, arm: str, sources: Path, names: list[str], md_set: Path, md_ids: list[str], wd_set: Path,
|
|
wiki: Path | None = None, astro: Path | None = None) -> list[Path]:
|
|
if arm not in ARMS:
|
|
raise ProbeError(f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}")
|
|
if arm == "wiki" and wiki is None:
|
|
raise ProbeError("the wiki arm needs --wiki")
|
|
if arm == "astro" and astro is None:
|
|
raise ProbeError("the astro arm needs --astro")
|
|
questions = probe_questions(md_set, md_ids, wd_set)
|
|
basis = run / "arms" / arm
|
|
if basis.exists():
|
|
raise ProbeError(f"{basis} exists already; an arm is set up once, from fresh")
|
|
if arm == "filsok":
|
|
copy_names(sources, names, basis)
|
|
elif arm == "wiki" and wiki is not None:
|
|
shutil.copytree(wiki, basis / "wiki")
|
|
copy_names(sources, names, basis / "kilde")
|
|
elif astro is not None:
|
|
shutil.copytree(astro / "raw", basis / "raw")
|
|
shutil.copytree(astro / "wiki", basis / "wiki")
|
|
tasks = []
|
|
for group, q in questions:
|
|
key = f"{group}-{q['id']}"
|
|
task = run / "tasks" / arm / f"{key}.md"
|
|
answer = run / "answers" / arm / f"{key}.json"
|
|
task.parent.mkdir(parents=True, exist_ok=True)
|
|
task.write_text(render_task(arm, basis, q["id"], q["question"], q.get("type") == "count", answer),
|
|
encoding="utf-8")
|
|
tasks.append(task)
|
|
(run / "answers" / arm).mkdir(parents=True, exist_ok=True)
|
|
return tasks
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- collect
|
|
|
|
|
|
def collect(run: Path, arm: str) -> tuple[Path, Path, list[str]]:
|
|
"""The per-question answers into one answers file per gate. The id comes from the file name."""
|
|
folder = run / "answers" / arm
|
|
gathered: dict[str, list[dict[str, Any]]] = {"md": [], "wd": []}
|
|
problems = []
|
|
for path in sorted(folder.glob("*.json")):
|
|
group, _, qid = path.stem.partition("-")
|
|
if group not in gathered or not qid:
|
|
problems.append(f"{path.name}: not a probe answer name")
|
|
continue
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
|
data = None
|
|
if not isinstance(data, dict):
|
|
problems.append(f"{path.name}: not a JSON object")
|
|
continue
|
|
gathered[group].append({**data, "id": qid})
|
|
md_path, wd_path = run / "answers" / f"{arm}.md.json", run / "answers" / f"{arm}.wd.json"
|
|
write_json(md_path, {"arm": arm, "answers": gathered["md"]})
|
|
write_json(wd_path, {"arm": arm, "answers": gathered["wd"]})
|
|
return md_path, wd_path, problems
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- harvest
|
|
|
|
|
|
def first_prompt(lines: list[dict[str, Any]]) -> tuple[str, str]:
|
|
for line in lines:
|
|
if line.get("type") == "user":
|
|
content = line.get("message", {}).get("content")
|
|
if isinstance(content, list):
|
|
content = " ".join(b.get("text", "") for b in content if isinstance(b, dict))
|
|
return (content if isinstance(content, str) else ""), str(line.get("timestamp", ""))
|
|
return "", ""
|
|
|
|
|
|
def inside(path: object, root: Path) -> bool:
|
|
if not isinstance(path, str) or not os.path.isabs(path):
|
|
return False
|
|
norm = Path(os.path.normpath(path))
|
|
return norm == root or root in norm.parents
|
|
|
|
|
|
def bash_breach(command: str, basis: Path) -> str | None:
|
|
"""None when the command is read-only search that starts with `cd <basis> &&` and stays inside the basis."""
|
|
if "`" in command or re.search(r"\$[A-Za-z_{(]", command):
|
|
return "Bash with an expansion"
|
|
try:
|
|
lexer = shlex.shlex(re.sub(r"\s2>/dev/null(?=\s|$)", "", command), posix=True, punctuation_chars="|&;<>()")
|
|
lexer.whitespace_split = True
|
|
tokens = list(lexer)
|
|
except ValueError:
|
|
return "Bash that cannot be parsed"
|
|
segments: list[list[str]] = [[]]
|
|
for token in tokens:
|
|
if token in SEPARATORS:
|
|
segments.append([])
|
|
elif token and set(token) <= set("&<>()"):
|
|
return f"Bash with {token!r}"
|
|
else:
|
|
segments[-1].append(token)
|
|
first = segments[0]
|
|
if len(first) != 2 or first[0] != "cd" or not inside(first[1], basis):
|
|
return "Bash that does not start with cd into the basis"
|
|
for segment in segments[1:]:
|
|
if not segment or segment[0] not in READ_ONLY:
|
|
return f"Bash command not allowed: {segment[0] if segment else '(empty)'}"
|
|
for token in segment[1:]:
|
|
value = token.split("=", 1)[1] if token.startswith("-") and "=" in token else token
|
|
if segment[0] == "find" and token in FIND_ACTIONS:
|
|
return f"Bash find action {token}"
|
|
if segment[0] == "sed" and (token.startswith("-i") or token.startswith("--in-place")):
|
|
return "Bash sed -i"
|
|
if value.startswith("~") or ".." in value.split("/"):
|
|
return f"Bash path outside the basis: {token}"
|
|
if value.startswith("/") and value != "/dev/null" and not inside(value, basis):
|
|
return f"Bash path outside the basis: {token}"
|
|
return None
|
|
|
|
|
|
def breaches_of(tools: list[tuple[str, dict[str, Any]]], basis: Path, task: Path, answer: Path) -> list[str]:
|
|
found = []
|
|
for name, inp in tools:
|
|
if name not in ALLOWED_TOOLS:
|
|
found.append(f"{name}: tool not allowed")
|
|
elif name == "Read":
|
|
path = inp.get("file_path")
|
|
if not (inside(path, basis) or (isinstance(path, str) and os.path.normpath(path) == str(task))):
|
|
found.append(f"Read outside the basis: {path}")
|
|
elif name in ("Grep", "Glob"):
|
|
path = inp.get("path")
|
|
pattern = inp.get("pattern", "")
|
|
if path is None:
|
|
found.append(f"{name} without a path (searches the working directory)")
|
|
elif not inside(path, basis):
|
|
found.append(f"{name} outside the basis: {path}")
|
|
elif name == "Glob" and isinstance(pattern, str) and os.path.isabs(pattern) and not inside(pattern, basis):
|
|
found.append(f"Glob pattern outside the basis: {pattern}")
|
|
elif name == "Bash":
|
|
breach = bash_breach(str(inp.get("command", "")), basis)
|
|
if breach:
|
|
found.append(breach)
|
|
elif name == "Write" and inp.get("file_path") != str(answer):
|
|
found.append(f"Write to another file: {inp.get('file_path')}")
|
|
return found
|
|
|
|
|
|
def read_transcript(path: Path) -> list[dict[str, Any]]:
|
|
lines = []
|
|
for raw in path.read_text(encoding="utf-8").splitlines():
|
|
try:
|
|
line = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if isinstance(line, dict):
|
|
lines.append(line)
|
|
return lines
|
|
|
|
|
|
def harvest(transcripts: Path, run: Path) -> dict[str, dict[str, Any]]:
|
|
"""Per task: all tokens its transcript records (each message id once), and every isolation breach."""
|
|
pattern = re.compile(re.escape(str(run / "tasks")) + r"/(" + "|".join(ARMS) + r")/((?:md|wd)-[A-Za-z0-9]+)\.md")
|
|
rows: dict[str, dict[str, Any]] = {}
|
|
for path in sorted(transcripts.glob("*.jsonl")):
|
|
lines = read_transcript(path)
|
|
prompt, started = first_prompt(lines)
|
|
match = pattern.search(prompt)
|
|
if match is None:
|
|
continue
|
|
arm, key = match.group(1), match.group(2)
|
|
usage: dict[str, dict[str, int]] = {}
|
|
tools: list[tuple[str, dict[str, Any]]] = []
|
|
for line in lines:
|
|
message = line.get("message", {})
|
|
if line.get("type") != "assistant" or not isinstance(message, dict):
|
|
continue
|
|
mid = str(message.get("id", ""))
|
|
fields = message.get("usage") or {}
|
|
seen = usage.setdefault(mid, {})
|
|
for f in USAGE_FIELDS:
|
|
value = fields.get(f)
|
|
if isinstance(value, int):
|
|
seen[f] = max(seen.get(f, 0), value)
|
|
for block in message.get("content") or []:
|
|
if isinstance(block, dict) and block.get("type") == "tool_use":
|
|
tools.append((str(block.get("name")), block.get("input") or {}))
|
|
row = {
|
|
"tokens": sum(sum(v.values()) for v in usage.values()),
|
|
"breaches": breaches_of(tools, run / "arms" / arm, run / "tasks" / arm / f"{key}.md",
|
|
run / "answers" / arm / f"{key}.json"),
|
|
"transcript": path.name,
|
|
"started": started,
|
|
}
|
|
name = f"{arm}/{key}"
|
|
previous = rows.get(name)
|
|
runs = 1 + (previous["runs"] if previous else 0)
|
|
if previous is None or started >= previous["started"]:
|
|
rows[name] = {**row, "runs": runs}
|
|
else:
|
|
previous["runs"] = runs
|
|
for row in rows.values():
|
|
row.pop("started")
|
|
return rows
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- score
|
|
|
|
|
|
def gate_results(path: Path) -> dict[str, dict[str, Any]]:
|
|
if not path.is_file():
|
|
raise ProbeError(f"no gate results at {path}; an arm that was not scored is a failure, not a zero")
|
|
data = load_json(path)
|
|
return {r["id"]: r for r in data.get("results", [])}
|
|
|
|
|
|
def mcnemar_exact(only_a: int, only_b: int) -> float:
|
|
"""Two-sided exact McNemar: the binomial test over the discordant pairs, p = 0.5."""
|
|
n, k = only_a + only_b, min(only_a, only_b)
|
|
if n == 0:
|
|
return 1.0
|
|
return min(1.0, 2 * sum(comb(n, i) for i in range(k + 1)) / float(2 ** n))
|
|
|
|
|
|
def score(run: Path, arms: list[str], md_ids: list[str]) -> dict[str, Any]:
|
|
tokens_path = run / "tokens.json"
|
|
tokens = load_json(tokens_path) if tokens_path.is_file() else {}
|
|
correct: dict[str, dict[str, dict[str, bool]]] = {}
|
|
table: dict[str, Any] = {"arms": {}, "pairs": {}, "mcnemar": {}}
|
|
for arm in arms:
|
|
md = gate_results(run / "results" / f"{arm}.md.json")
|
|
wd = gate_results(run / "results" / f"{arm}.wd.json")
|
|
by_group: dict[str, dict[str, bool]] = {g: {} for g in GROUPS}
|
|
answer_ok = source_ok = 0
|
|
for qid in md_ids:
|
|
r = md.get(qid, {})
|
|
by_group["flerdokument"][f"md-{qid}"] = bool(r.get("correct"))
|
|
answer_ok += bool(r.get("answer_ok"))
|
|
source_ok += bool(r.get("source_ok"))
|
|
for qid, r in wd.items():
|
|
group = "telling" if r.get("type") == "count" else "helt dokument"
|
|
by_group[group][f"wd-{qid}"] = bool(r.get("correct"))
|
|
answer_ok += bool(r.get("answer_ok"))
|
|
source_ok += bool(r.get("source_ok"))
|
|
correct[arm] = by_group
|
|
n = sum(len(v) for v in by_group.values())
|
|
own = [v for k, v in tokens.items() if k.startswith(f"{arm}/")]
|
|
table["arms"][arm] = {
|
|
"groups": {g: [sum(v.values()), len(v)] for g, v in by_group.items()},
|
|
"total": [sum(sum(v.values()) for v in by_group.values()), n],
|
|
"answer_ok": answer_ok,
|
|
"source_ok": source_ok,
|
|
"tokens": {"sum": sum(int(v["tokens"]) for v in own), "questions": len(own)},
|
|
"breaches": sum(len(v["breaches"]) for v in own),
|
|
}
|
|
for a, b in combinations(arms, 2):
|
|
a, b = (b, a) if b == "wiki" else (a, b)
|
|
pair: dict[str, dict[str, int]] = {}
|
|
for g in GROUPS:
|
|
counts = {"both": 0, f"only {a}": 0, f"only {b}": 0, "neither": 0}
|
|
for key, ok_a in correct[a][g].items():
|
|
ok_b = correct[b][g].get(key, False)
|
|
label = "both" if ok_a and ok_b else f"only {a}" if ok_a else f"only {b}" if ok_b else "neither"
|
|
counts[label] += 1
|
|
pair[g] = counts
|
|
table["pairs"][f"{a} vs {b}"] = pair
|
|
only_a, only_b = sum(c[f"only {a}"] for c in pair.values()), sum(c[f"only {b}"] for c in pair.values())
|
|
table["mcnemar"][f"{a} vs {b}"] = {f"only {a}": only_a, f"only {b}": only_b, "p": mcnemar_exact(only_a, only_b)}
|
|
table["per_question"] = {arm: {k: v for g in GROUPS for k, v in correct[arm][g].items()} for arm in arms}
|
|
return table
|
|
|
|
|
|
def render_table(table: dict[str, Any]) -> str:
|
|
head = "| arm | " + " | ".join(f"{g} (N)" for g in GROUPS) + " | total | answer ok | source ok | tokens/question |"
|
|
lines = [head, "|" + "---|" * (len(GROUPS) + 5)]
|
|
for arm, row in table["arms"].items():
|
|
t = row["tokens"]
|
|
per_q = f"{t['sum'] / t['questions'] / 1e6:.2f} M" if t["questions"] else "not harvested"
|
|
cells = [f"{c} of {n}" for c, n in (row["groups"][g] for g in GROUPS)]
|
|
total = row["total"]
|
|
lines.append(f"| {arm} | " + " | ".join(cells) + f" | {total[0]} of {total[1]} | {row['answer_ok']} | "
|
|
f"{row['source_ok']} | {per_q} |")
|
|
lines.append("")
|
|
for name, pair in table["pairs"].items():
|
|
lines.append(f"{name}: " + "; ".join(f"{g}: " + ", ".join(f"{k} {v}" for k, v in c.items())
|
|
for g, c in pair.items()))
|
|
m = table["mcnemar"][name]
|
|
lines.append(f"{name}, all groups: " + ", ".join(f"{k} {v}" for k, v in m.items() if k != "p")
|
|
+ f"; exact McNemar p = {m['p']:.3g}")
|
|
return "\n".join(lines)
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- main
|
|
|
|
|
|
def cut_names(manifest: Path) -> list[str]:
|
|
names = load_json(manifest).get("cut", {}).get("names")
|
|
if not isinstance(names, list) or not names:
|
|
raise ProbeError(f"{manifest} records no cut names")
|
|
return [str(n) for n in names]
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
f = sub.add_parser("fetch-astro")
|
|
f.add_argument("--sources", required=True)
|
|
f.add_argument("--cut-manifest", required=True)
|
|
f.add_argument("--project", required=True)
|
|
f.add_argument("--topic", required=True)
|
|
f.add_argument("--collected", required=True)
|
|
s = sub.add_parser("setup")
|
|
s.add_argument("--run", required=True)
|
|
s.add_argument("--arm", required=True)
|
|
s.add_argument("--sources", required=True)
|
|
s.add_argument("--cut-manifest", required=True)
|
|
s.add_argument("--md-set", required=True)
|
|
s.add_argument("--md-ids", required=True, help="comma-separated")
|
|
s.add_argument("--wd-set", required=True)
|
|
s.add_argument("--wiki")
|
|
s.add_argument("--astro")
|
|
c = sub.add_parser("collect")
|
|
c.add_argument("--run", required=True)
|
|
c.add_argument("--arm", required=True)
|
|
h = sub.add_parser("harvest")
|
|
h.add_argument("--run", required=True)
|
|
h.add_argument("--transcripts", required=True)
|
|
r = sub.add_parser("score")
|
|
r.add_argument("--run", required=True)
|
|
r.add_argument("--arms", required=True, help="comma-separated")
|
|
r.add_argument("--md-ids", required=True, help="comma-separated")
|
|
return p
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
try:
|
|
if args.command == "fetch-astro":
|
|
written = fetch_astro(Path(args.sources), cut_names(Path(args.cut_manifest)), Path(args.project),
|
|
args.topic, args.collected)
|
|
print(f"raw files: {len(written)}")
|
|
elif args.command == "setup":
|
|
tasks = setup_arm(Path(args.run), args.arm, Path(args.sources), cut_names(Path(args.cut_manifest)),
|
|
Path(args.md_set), args.md_ids.split(","), Path(args.wd_set),
|
|
Path(args.wiki) if args.wiki else None, Path(args.astro) if args.astro else None)
|
|
print("\n".join(str(t) for t in tasks))
|
|
elif args.command == "collect":
|
|
md_path, wd_path, problems = collect(Path(args.run), args.arm)
|
|
print(f"{md_path}\n{wd_path}")
|
|
for problem in problems:
|
|
print(f"problem: {problem}")
|
|
elif args.command == "harvest":
|
|
rows = harvest(Path(args.transcripts), Path(args.run))
|
|
write_json(Path(args.run) / "tokens.json", rows)
|
|
print(f"tasks harvested: {len(rows)}; breaches: {sum(len(r['breaches']) for r in rows.values())}")
|
|
else:
|
|
table = score(Path(args.run), args.arms.split(","), args.md_ids.split(","))
|
|
write_json(Path(args.run) / "table.json", table)
|
|
print(render_table(table))
|
|
except ProbeError as exc:
|
|
print(f"FAIL: {exc}", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|