okf-wiki/tests/v1_candidate.sh
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

227 lines
11 KiB
Bash
Executable file

#!/bin/bash
# v1 run-output checks (V1 Step 3, rewritten in wave 2 order 7 for plan revision 5, .claude/projects/2026-10-03-v1-k):
# the same thirteen checks on the first part (del1/, Steps 11-14) and on collection 1 (samling1/, Steps 15-19), and two
# once. One PASS or FAIL line per check; a check that cannot run prints FAIL. Exit 1 on any FAIL. bash 3.2 safe, ASCII.
# V1_ROOT holds del1/ and samling1/ (default $HOME/okf/benk/v1, where Step 11 puts the cost line). Each run has its own
# source root with kilde/ and source-manifest.json (K1): DEL1_SOURCES (default $HOME/okf/benk/samling1/del1) and
# SAMLING1_SOURCES (default $HOME/okf/benk/samling1/20261008). OKF_UPSTREAM is the mirror of Google's repository
# (default $HOME/repos/_okf-canonical), read only through git archive of OKF_UPSTREAM_REV (default ad30107, the pinned
# commit) into a temporary folder, and never between 07:10 and 07:25, when the daily job fetches it. V1_PROJECT is the
# plan's project folder, which holds review.md (default .claude/projects/2026-10-03-v1-k, local).
# Every line reads a folder under V1_ROOT or V1_PROJECT first, so with both empty every line is FAIL.
# The answer measurement writes its result files (contract K5, okf-wiki/answer-results/1) to <run>/svar/filsok.json and
# <run>/svar/bundle.json; the change warning writes <run>/endring/diff.json (okf-wiki/source-diff/1) and marks.json
# (okf-wiki/source-marks/1) for samling1.
#
# Expected state of each line after wave 2 order 7, and the plan step that turns it PASS. Every line is FAIL now,
# because no v1 run folder exists. tests/test_scripts.py checks that this table and the check lines below agree.
# <run> criterion 1 del1 -> Step 11, samling1 -> Step 15
# <run> link rule del1 -> Step 11, samling1 -> Step 15
# <run> auditor exits 0 del1 -> Step 11, samling1 -> Step 15
# <run> view render del1 -> Step 11, samling1 -> Step 15
# <run> view versions del1 -> Step 11, samling1 -> Step 15
# <run> kost.json del1 -> Step 11, samling1 -> Step 15
# <run> criterion 3 del1 -> Step 14, samling1 -> Step 19
# <run> criterion 2 del1 -> Step 12, samling1 -> Step 16
# <run> Google viewer del1 -> Step 11, samling1 -> Step 18a
# <run> every x- field del1 -> Step 11, samling1 -> Step 18a
# <run> isolation breaks 0 del1 -> Step 12, samling1 -> Step 16 (isolation.json key breaks)
# <run> lesing.json links del1 -> Step 14, samling1 -> Step 19
# <run> criterion 4 del1 -> Step 13 (both arms measured), samling1 -> Step 17 (bundle better, p < 0.05)
# change warning Step 18a (the frozen copy against a fresh fetch, marks shown)
# review Step 18b (review.md frontmatter type trekreview, Remediation Summary BLOCKER: 0)
set -u
cd "$(dirname "$0")/.." || exit 2
V1_ROOT="${V1_ROOT:-$HOME/okf/benk/v1}"
DEL1_SOURCES="${DEL1_SOURCES:-$HOME/okf/benk/samling1/del1}"
SAMLING1_SOURCES="${SAMLING1_SOURCES:-$HOME/okf/benk/samling1/20261008}"
OKF_UPSTREAM="${OKF_UPSTREAM:-$HOME/repos/_okf-canonical}"
OKF_UPSTREAM_REV="${OKF_UPSTREAM_REV:-ad30107}"
V1_PROJECT="${V1_PROJECT:-.claude/projects/2026-10-03-v1-k}"
export V1_ROOT DEL1_SOURCES SAMLING1_SOURCES OKF_UPSTREAM OKF_UPSTREAM_REV V1_PROJECT
FAILED=0
check() {
name="$1"
shift
if "$@" >/dev/null 2>&1; then
echo "PASS $name"
else
echo "FAIL $name"
FAILED=1
fi
}
py() {
python3 -c "$1"
}
has_bundle() {
[ -f "$V1_ROOT/$RUN/index.md" ]
}
criterion_1() {
has_bundle || return 1
out="$(mktemp -d)" || return 1
python3 -m okfwiki.claims list "$V1_ROOT/$RUN" "$V1_SOURCES" --out "$out/claims.json"
rc=$?
rm -rf "$out"
return "$rc"
}
auditor_twice() {
has_bundle || return 1
out="$(mktemp -d)" || return 1
python3 -m okfwiki.audit "$V1_ROOT/$RUN" "$V1_SOURCES" >"$out/a.json"
rc=$?
python3 -m okfwiki.audit "$V1_ROOT/$RUN" "$V1_SOURCES" >"$out/b.json"
[ "$rc" -eq 0 ] && cmp -s "$out/a.json" "$out/b.json"
rc=$?
rm -rf "$out"
return "$rc"
}
view_check() {
has_bundle || return 1
out="$(mktemp -d)" || return 1
python3 -m okfwiki.view render "$V1_ROOT/$RUN" "$V1_SOURCES" --out "$out/html" &&
python3 -m okfwiki.view check "$V1_ROOT/$RUN" "$out/html"
rc=$?
rm -rf "$out"
return "$rc"
}
view_versions_overlap() {
has_bundle || return 1
python3 -m okfwiki.view versions "$V1_ROOT/$RUN" "$V1_SOURCES" &&
python3 -m okfwiki.view overlap "$V1_ROOT/$RUN" "$V1_SOURCES"
}
criterion_2() {
[ -f "$V1_ROOT/$RUN/claims.json" ] && [ -f "$V1_ROOT/$RUN/verdicts.json" ] || return 1
python3 -m okfwiki.claims score --claims "$V1_ROOT/$RUN/claims.json" --verdicts "$V1_ROOT/$RUN/verdicts.json"
}
google_viewer() {
has_bundle && [ -f tools/google_viewer_check.py ] || return 1
hm=$((10#$(date +%H%M)))
if [ "$hm" -ge 710 ] && [ "$hm" -le 725 ]; then
return 1
fi
out="$(mktemp -d)" || return 1
git -C "$OKF_UPSTREAM" archive "$OKF_UPSTREAM_REV" | tar -x -C "$out" &&
python3 tools/google_viewer_check.py "$V1_ROOT/$RUN" --upstream "$out"
rc=$?
rm -rf "$out"
return "$rc"
}
x_fields() {
has_bundle && [ -f tools/x_fields_check.py ] || return 1
python3 tools/x_fields_check.py "$V1_ROOT/$RUN" --table docs/x-fields.md
}
change_warning() {
RUN=samling1
export RUN
V1_SOURCES="$SAMLING1_SOURCES"
export V1_SOURCES
has_bundle || return 1
d="$V1_ROOT/samling1/endring"
[ -f "$d/diff.json" ] && [ -f "$d/marks.json" ] || return 1
py '
import hashlib, json, os, sys
d = os.path.join(os.environ["V1_ROOT"], "samling1", "endring")
k1 = os.path.join(os.environ["SAMLING1_SOURCES"], "source-manifest.json")
diff = json.load(open(os.path.join(d, "diff.json")))
marks = json.load(open(os.path.join(d, "marks.json")))
frozen = hashlib.sha256(open(k1, "rb").read()).hexdigest() if os.path.isfile(k1) else ""
old = diff.get("old") if isinstance(diff, dict) else None
ok = (isinstance(old, dict) and diff.get("schema") == "okf-wiki/source-diff/1" and frozen and old.get("manifest_sha256") == frozen
and isinstance(marks, dict) and marks.get("schema") == "okf-wiki/source-marks/1")
sys.exit(0 if ok else 1)' || return 1
out="$(mktemp -d)" || return 1
python3 -m okfwiki.view render "$V1_ROOT/samling1" "$V1_SOURCES" --out "$out/html" --marks "$d/marks.json"
rc=$?
rm -rf "$out"
return "$rc"
}
for RUN in del1 samling1; do
export RUN
case "$RUN" in
del1) V1_SOURCES="$DEL1_SOURCES" ;;
*) V1_SOURCES="$SAMLING1_SOURCES" ;;
esac
export V1_SOURCES
check "$RUN criterion 1: okfwiki.claims list exits 0" criterion_1
check "$RUN link rule: main pages N of N sources, linked from a main page N of N pages, 0 misplaced, 0 broken" py '
import json, os, subprocess, sys
run = os.path.join(os.environ["V1_ROOT"], os.environ["RUN"])
if not os.path.isfile(os.path.join(run, "index.md")):
sys.exit(1)
res = subprocess.run([sys.executable, "-m", "okfwiki.audit", run, os.environ["V1_SOURCES"]], capture_output=True, text=True)
if res.returncode not in (0, 1):
sys.exit(1)
out = json.loads(res.stdout)
rule = out.get("link_rule")
if not isinstance(rule, dict):
sys.exit(1)
main, linked = rule.get("main_pages"), rule.get("linked")
if not isinstance(main, dict) or not isinstance(linked, dict) or not isinstance(main.get("of"), int) or main["of"] < 1:
sys.exit(1)
classes = ("link-misplaced", "broken-link", "main-page-missing", "main-page-duplicate", "main-page-invalid", "no-main-page-link")
bad = [f for f in out.get("findings", []) if f.get("check") in classes]
sys.exit(0 if main.get("pass") == main["of"] and linked.get("pass") == linked.get("of") and not bad else 1)'
check "$RUN auditor exits 0 and two runs give identical bytes" auditor_twice
check "$RUN view render and view check exit 0" view_check
check "$RUN view versions and view overlap report 0" view_versions_overlap
check "$RUN kost.json has a margin and cost summarize is within its cap" py '
import json, os, re, subprocess, sys
run = os.path.join(os.environ["V1_ROOT"], os.environ["RUN"])
kost = os.path.join(run, "kost.json")
if not os.path.isfile(kost) or "margin" not in json.load(open(kost)):
sys.exit(1)
res = subprocess.run([sys.executable, "-m", "okfwiki.cost", "summarize", "--kost", kost], capture_output=True, text=True)
lines = [m for m in (re.search(r"= ([0-9]+) of ([0-9]+)", x) for x in res.stdout.splitlines()) if m]
sys.exit(0 if res.returncode == 0 and lines and all(int(m.group(1)) <= int(m.group(2)) for m in lines) else 1)'
check "$RUN criterion 3: lesing.json says 20 of 20 with the draw sha256" py '
import json, os, re, sys
f = os.path.join(os.environ["V1_ROOT"], os.environ["RUN"], "lesing.json")
d = json.load(open(f)) if os.path.isfile(f) else {}
sys.exit(0 if d.get("answer") == "20 of 20" and re.fullmatch(r"[0-9a-f]{64}", str(d.get("sha256", ""))) else 1)'
check "$RUN criterion 2: okfwiki.claims score exits 0" criterion_2
check "$RUN Google viewer reads the bundle (tools/google_viewer_check.py)" google_viewer
check "$RUN every x- field has a measured reason (tools/x_fields_check.py)" x_fields
check "$RUN isolation breaks 0: each reader reads only its task file" py '
import json, os, sys
f = os.path.join(os.environ["V1_ROOT"], os.environ["RUN"], "isolation.json")
d = json.load(open(f)) if os.path.isfile(f) else {}
sys.exit(0 if d.get("breaks") == 0 else 1)'
check "$RUN lesing.json links: a links yes/no for each of the 20 pages" py '
import json, os, sys
f = os.path.join(os.environ["V1_ROOT"], os.environ["RUN"], "lesing.json")
pages = (json.load(open(f)) if os.path.isfile(f) else {}).get("pages")
sys.exit(0 if isinstance(pages, list) and len(pages) == 20 and all(isinstance(p, dict) and p.get("links") in ("yes", "no") for p in pages) else 1)'
check "$RUN criterion 4: both arms answered the same questions (K5); on samling1 the bundle arm is better, McNemar p < 0.05" py '
import json, math, os, sys
d = os.path.join(os.environ["V1_ROOT"], os.environ["RUN"], "svar")
arms = {}
for arm in ("filsok", "bundle"):
f = os.path.join(d, arm + ".json")
x = json.load(open(f)) if os.path.isfile(f) else {}
res = x.get("results") if isinstance(x, dict) else None
if x.get("schema") != "okf-wiki/answer-results/1" or x.get("arm") != arm or not isinstance(res, list) or not res:
sys.exit(1)
if not all(isinstance(r, dict) and isinstance(r.get("id"), str) and isinstance(r.get("correct"), bool) for r in res):
sys.exit(1)
arms[arm] = (x.get("set_sha256"), {r["id"]: r["correct"] for r in res})
(set_a, a), (set_b, b) = arms["filsok"], arms["bundle"]
if not set_a or set_a != set_b or set(a) != set(b):
sys.exit(1)
if os.environ["RUN"] != "samling1":
sys.exit(0)
only_a = sum(1 for i in a if a[i] and not b[i])
only_b = sum(1 for i in a if b[i] and not a[i])
n = only_a + only_b
p = min(1.0, 2 * sum(math.comb(n, k) for k in range(min(only_a, only_b) + 1)) / 2 ** n) if n else 1.0
sys.exit(0 if only_b > only_a and p < 0.05 else 1)'
done
check "change warning: the frozen copy of samling1 against a fresh fetch, marks shown in the view" change_warning
check "review: 0 BLOCKER (SC19)" py '
import os, re, sys
f = os.path.join(os.environ["V1_PROJECT"], "review.md")
text = open(f, encoding="utf-8").read() if os.path.isfile(f) else ""
front = re.match(r"---\n(.*?)\n---\n", text, re.S)
count = re.search(r"^- BLOCKER: ([0-9]+)$", text, re.M)
sys.exit(0 if front and re.search(r"^type: trekreview$", front.group(1), re.M) and count and count.group(1) == "0" else 1)'
exit "$FAILED"