First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
96 lines
4.3 KiB
Bash
96 lines
4.3 KiB
Bash
#!/bin/bash
|
|
# TRIAL (P1 method trial, branch p1-metodeprove): not v1 code.
|
|
# The trial's output checks: SC1, SC2, SC3, coverage and SC6, SC7, SC8, the SC10 table, and rule 3 for the test
|
|
# fixtures. P1_ROOT is the folder that contains kilde/ and prove-p1/ (default $HOME/okf/benk/wiki, because the
|
|
# criteria runner passes no arguments; the path names no content). One PASS or FAIL line per check; a check that
|
|
# cannot run is a FAIL. Exit 1 on any FAIL. bash 3.2 safe, ASCII only.
|
|
set -u
|
|
cd "$(dirname "$0")/.." || exit 2
|
|
ROOT="${P1_ROOT:-$HOME/okf/benk/wiki}"
|
|
P="$ROOT/prove-p1"
|
|
export ROOT P
|
|
FAILED=0
|
|
check() {
|
|
name="$1"
|
|
shift
|
|
if "$@" >/dev/null 2>&1; then
|
|
echo "PASS $name"
|
|
else
|
|
echo "FAIL $name"
|
|
FAILED=1
|
|
fi
|
|
}
|
|
py() {
|
|
python3 -c "$1"
|
|
}
|
|
|
|
check "SC1 K, K+ and D each built or stopped" py '
|
|
import json, os, sys
|
|
p = os.environ["P"]
|
|
stops = json.load(open(p + "/kost.json")).get("stops", {}) if os.path.isfile(p + "/kost.json") else {}
|
|
sys.exit(0 if all(os.path.isdir(p + "/" + v) or v in stops for v in ("K", "K+", "D")) else 1)'
|
|
check "SC1 rekkefolge.txt has 19 sources" py '
|
|
import os, sys
|
|
f = os.environ["P"] + "/rekkefolge.txt"
|
|
sys.exit(0 if os.path.isfile(f) and len(open(f).read().split()) == 19 else 1)'
|
|
check "SC7 begreper.txt has 3 concepts" py '
|
|
import os, sys
|
|
f = os.environ["P"] + "/begreper.txt"
|
|
sys.exit(0 if os.path.isfile(f) and len([x for x in open(f) if x.strip()]) == 3 else 1)'
|
|
check "SC2 kost.json: four categories, startup, turns, subagents, means and projections" py '
|
|
import json, os, sys
|
|
d = json.load(open(os.environ["P"] + "/kost.json"))
|
|
k = d["variants"]["K"]
|
|
need = ("input", "cache_read", "cache_write", "output", "startup", "turns", "subagents")
|
|
ok = d["schema"] == "okf-wiki/p1-cost/1" and all(x in k for x in need)
|
|
ok = ok and set(k["means"]) == {"1-5", "15-19"} and k["projection"]["fit"] and k["projection"]["flat"]
|
|
sys.exit(0 if ok else 1)'
|
|
check "SC3 resultat.json: audit exit and criterion 1 per built variant" py '
|
|
import json, os, sys
|
|
p = os.environ["P"]
|
|
d = json.load(open(p + "/resultat.json"))["variants"]
|
|
built = [v for v in ("K", "K+", "D") if os.path.isdir(p + "/" + v)]
|
|
sys.exit(0 if built and all("exit" in d[v] and "criterion_1" in d[v] for v in built) else 1)'
|
|
check "SC5b and SC6: coverage and structure per built variant" py '
|
|
import json, os, sys
|
|
p = os.environ["P"]
|
|
built = [v for v in ("K", "K+", "D") if os.path.isdir(p + "/" + v)]
|
|
need = ("concepts_per_source", "skeleton_unused", "duplicates", "islands", "precision", "quotes")
|
|
ok = bool(built)
|
|
for v in built:
|
|
s = json.load(open(p + "/struktur-" + v + ".json"))
|
|
c = json.load(open(p + "/dekning-" + v + ".json"))
|
|
ok = ok and all(x in s for x in need) and "covered" in c and "total" in c
|
|
sys.exit(0 if ok else 1)'
|
|
check "SC7 every source link on the side-by-side page resolves" env PYTHONPATH=evals:. python3 -m p1.sidebyside check --html "$P/side-om-side.html" --kilde-root "$ROOT"
|
|
check "SC8 the reference bundles are untouched since .start" py '
|
|
import os, subprocess, sys
|
|
p, root = os.environ["P"], os.environ["ROOT"]
|
|
dirs = [root + "/" + d for d in ("utsnitt-l2b", "utsnitt-l2b-html", "utsnitt-l4", "utsnitt-l4-html")]
|
|
if not os.path.isfile(p + "/.start") or not all(os.path.isdir(d) for d in dirs):
|
|
sys.exit(1)
|
|
out = subprocess.run(["find", *dirs, "-newer", p + "/.start"], capture_output=True, text=True, check=True).stdout
|
|
sys.exit(0 if out.strip() == "" else 1)'
|
|
check "SC10 rapport.md has the table with every column" py '
|
|
import os, sys
|
|
f = os.environ["P"] + "/rapport.md"
|
|
cols = ("built to source", "concepts", "skeleton unused", "merge candidates", "coverage", "tokens per source",
|
|
"1-5", "15-19", "turns", "projected 599", "criterion 1", "code quotes", "precision", "islands", "duplicates")
|
|
sys.exit(0 if os.path.isfile(f) and all(c in open(f).read() for c in cols) else 1)'
|
|
check "rule 3: no 8-word run of a test fixture is in kilde/" py '
|
|
import glob, os, re, sys
|
|
kilde = os.environ["ROOT"] + "/kilde"
|
|
files = glob.glob(kilde + "/*.md")
|
|
if not files:
|
|
sys.exit(1)
|
|
def runs(text):
|
|
w = re.findall(r"[a-z0-9]+", text.lower())
|
|
return {tuple(w[i:i + 8]) for i in range(len(w) - 7)}
|
|
mine = set()
|
|
for t in glob.glob("evals/tests/test_p1_*.py"):
|
|
mine |= runs(open(t).read())
|
|
src = set()
|
|
for f in files:
|
|
src |= runs(open(f, encoding="utf-8").read())
|
|
sys.exit(0 if not (mine & src) else 1)'
|
|
exit "$FAILED"
|