First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
227 lines
11 KiB
Bash
Executable file
227 lines
11 KiB
Bash
Executable file
#!/bin/bash
|
|
# v1 run-output checks (V1 Step 3, rewritten in wave 2 order 7 for plan revision 5, .claude/projects/2026-10-03-v1-k):
|
|
# the same thirteen checks on the first part (del1/, Steps 11-14) and on collection 1 (samling1/, Steps 15-19), and two
|
|
# once. One PASS or FAIL line per check; a check that cannot run prints FAIL. Exit 1 on any FAIL. bash 3.2 safe, ASCII.
|
|
# V1_ROOT holds del1/ and samling1/ (default $HOME/okf/benk/v1, where Step 11 puts the cost line). Each run has its own
|
|
# source root with kilde/ and source-manifest.json (K1): DEL1_SOURCES (default $HOME/okf/benk/samling1/del1) and
|
|
# SAMLING1_SOURCES (default $HOME/okf/benk/samling1/20261008). OKF_UPSTREAM is the mirror of Google's repository
|
|
# (default $HOME/repos/_okf-canonical), read only through git archive of OKF_UPSTREAM_REV (default ad30107, the pinned
|
|
# commit) into a temporary folder, and never between 07:10 and 07:25, when the daily job fetches it. V1_PROJECT is the
|
|
# plan's project folder, which holds review.md (default .claude/projects/2026-10-03-v1-k, local).
|
|
# Every line reads a folder under V1_ROOT or V1_PROJECT first, so with both empty every line is FAIL.
|
|
# The answer measurement writes its result files (contract K5, okf-wiki/answer-results/1) to <run>/svar/filsok.json and
|
|
# <run>/svar/bundle.json; the change warning writes <run>/endring/diff.json (okf-wiki/source-diff/1) and marks.json
|
|
# (okf-wiki/source-marks/1) for samling1.
|
|
#
|
|
# Expected state of each line after wave 2 order 7, and the plan step that turns it PASS. Every line is FAIL now,
|
|
# because no v1 run folder exists. tests/test_scripts.py checks that this table and the check lines below agree.
|
|
# <run> criterion 1 del1 -> Step 11, samling1 -> Step 15
|
|
# <run> link rule del1 -> Step 11, samling1 -> Step 15
|
|
# <run> auditor exits 0 del1 -> Step 11, samling1 -> Step 15
|
|
# <run> view render del1 -> Step 11, samling1 -> Step 15
|
|
# <run> view versions del1 -> Step 11, samling1 -> Step 15
|
|
# <run> kost.json del1 -> Step 11, samling1 -> Step 15
|
|
# <run> criterion 3 del1 -> Step 14, samling1 -> Step 19
|
|
# <run> criterion 2 del1 -> Step 12, samling1 -> Step 16
|
|
# <run> Google viewer del1 -> Step 11, samling1 -> Step 18a
|
|
# <run> every x- field del1 -> Step 11, samling1 -> Step 18a
|
|
# <run> isolation breaks 0 del1 -> Step 12, samling1 -> Step 16 (isolation.json key breaks)
|
|
# <run> lesing.json links del1 -> Step 14, samling1 -> Step 19
|
|
# <run> criterion 4 del1 -> Step 13 (both arms measured), samling1 -> Step 17 (bundle better, p < 0.05)
|
|
# change warning Step 18a (the frozen copy against a fresh fetch, marks shown)
|
|
# review Step 18b (review.md frontmatter type trekreview, Remediation Summary BLOCKER: 0)
|
|
set -u
|
|
cd "$(dirname "$0")/.." || exit 2
|
|
V1_ROOT="${V1_ROOT:-$HOME/okf/benk/v1}"
|
|
DEL1_SOURCES="${DEL1_SOURCES:-$HOME/okf/benk/samling1/del1}"
|
|
SAMLING1_SOURCES="${SAMLING1_SOURCES:-$HOME/okf/benk/samling1/20261008}"
|
|
OKF_UPSTREAM="${OKF_UPSTREAM:-$HOME/repos/_okf-canonical}"
|
|
OKF_UPSTREAM_REV="${OKF_UPSTREAM_REV:-ad30107}"
|
|
V1_PROJECT="${V1_PROJECT:-.claude/projects/2026-10-03-v1-k}"
|
|
export V1_ROOT DEL1_SOURCES SAMLING1_SOURCES OKF_UPSTREAM OKF_UPSTREAM_REV V1_PROJECT
|
|
FAILED=0
|
|
check() {
|
|
name="$1"
|
|
shift
|
|
if "$@" >/dev/null 2>&1; then
|
|
echo "PASS $name"
|
|
else
|
|
echo "FAIL $name"
|
|
FAILED=1
|
|
fi
|
|
}
|
|
py() {
|
|
python3 -c "$1"
|
|
}
|
|
has_bundle() {
|
|
[ -f "$V1_ROOT/$RUN/index.md" ]
|
|
}
|
|
criterion_1() {
|
|
has_bundle || return 1
|
|
out="$(mktemp -d)" || return 1
|
|
python3 -m okfwiki.claims list "$V1_ROOT/$RUN" "$V1_SOURCES" --out "$out/claims.json"
|
|
rc=$?
|
|
rm -rf "$out"
|
|
return "$rc"
|
|
}
|
|
auditor_twice() {
|
|
has_bundle || return 1
|
|
out="$(mktemp -d)" || return 1
|
|
python3 -m okfwiki.audit "$V1_ROOT/$RUN" "$V1_SOURCES" >"$out/a.json"
|
|
rc=$?
|
|
python3 -m okfwiki.audit "$V1_ROOT/$RUN" "$V1_SOURCES" >"$out/b.json"
|
|
[ "$rc" -eq 0 ] && cmp -s "$out/a.json" "$out/b.json"
|
|
rc=$?
|
|
rm -rf "$out"
|
|
return "$rc"
|
|
}
|
|
view_check() {
|
|
has_bundle || return 1
|
|
out="$(mktemp -d)" || return 1
|
|
python3 -m okfwiki.view render "$V1_ROOT/$RUN" "$V1_SOURCES" --out "$out/html" &&
|
|
python3 -m okfwiki.view check "$V1_ROOT/$RUN" "$out/html"
|
|
rc=$?
|
|
rm -rf "$out"
|
|
return "$rc"
|
|
}
|
|
view_versions_overlap() {
|
|
has_bundle || return 1
|
|
python3 -m okfwiki.view versions "$V1_ROOT/$RUN" "$V1_SOURCES" &&
|
|
python3 -m okfwiki.view overlap "$V1_ROOT/$RUN" "$V1_SOURCES"
|
|
}
|
|
criterion_2() {
|
|
[ -f "$V1_ROOT/$RUN/claims.json" ] && [ -f "$V1_ROOT/$RUN/verdicts.json" ] || return 1
|
|
python3 -m okfwiki.claims score --claims "$V1_ROOT/$RUN/claims.json" --verdicts "$V1_ROOT/$RUN/verdicts.json"
|
|
}
|
|
google_viewer() {
|
|
has_bundle && [ -f tools/google_viewer_check.py ] || return 1
|
|
hm=$((10#$(date +%H%M)))
|
|
if [ "$hm" -ge 710 ] && [ "$hm" -le 725 ]; then
|
|
return 1
|
|
fi
|
|
out="$(mktemp -d)" || return 1
|
|
git -C "$OKF_UPSTREAM" archive "$OKF_UPSTREAM_REV" | tar -x -C "$out" &&
|
|
python3 tools/google_viewer_check.py "$V1_ROOT/$RUN" --upstream "$out"
|
|
rc=$?
|
|
rm -rf "$out"
|
|
return "$rc"
|
|
}
|
|
x_fields() {
|
|
has_bundle && [ -f tools/x_fields_check.py ] || return 1
|
|
python3 tools/x_fields_check.py "$V1_ROOT/$RUN" --table docs/x-fields.md
|
|
}
|
|
change_warning() {
|
|
RUN=samling1
|
|
export RUN
|
|
V1_SOURCES="$SAMLING1_SOURCES"
|
|
export V1_SOURCES
|
|
has_bundle || return 1
|
|
d="$V1_ROOT/samling1/endring"
|
|
[ -f "$d/diff.json" ] && [ -f "$d/marks.json" ] || return 1
|
|
py '
|
|
import hashlib, json, os, sys
|
|
d = os.path.join(os.environ["V1_ROOT"], "samling1", "endring")
|
|
k1 = os.path.join(os.environ["SAMLING1_SOURCES"], "source-manifest.json")
|
|
diff = json.load(open(os.path.join(d, "diff.json")))
|
|
marks = json.load(open(os.path.join(d, "marks.json")))
|
|
frozen = hashlib.sha256(open(k1, "rb").read()).hexdigest() if os.path.isfile(k1) else ""
|
|
old = diff.get("old") if isinstance(diff, dict) else None
|
|
ok = (isinstance(old, dict) and diff.get("schema") == "okf-wiki/source-diff/1" and frozen and old.get("manifest_sha256") == frozen
|
|
and isinstance(marks, dict) and marks.get("schema") == "okf-wiki/source-marks/1")
|
|
sys.exit(0 if ok else 1)' || return 1
|
|
out="$(mktemp -d)" || return 1
|
|
python3 -m okfwiki.view render "$V1_ROOT/samling1" "$V1_SOURCES" --out "$out/html" --marks "$d/marks.json"
|
|
rc=$?
|
|
rm -rf "$out"
|
|
return "$rc"
|
|
}
|
|
|
|
for RUN in del1 samling1; do
|
|
export RUN
|
|
case "$RUN" in
|
|
del1) V1_SOURCES="$DEL1_SOURCES" ;;
|
|
*) V1_SOURCES="$SAMLING1_SOURCES" ;;
|
|
esac
|
|
export V1_SOURCES
|
|
check "$RUN criterion 1: okfwiki.claims list exits 0" criterion_1
|
|
check "$RUN link rule: main pages N of N sources, linked from a main page N of N pages, 0 misplaced, 0 broken" py '
|
|
import json, os, subprocess, sys
|
|
run = os.path.join(os.environ["V1_ROOT"], os.environ["RUN"])
|
|
if not os.path.isfile(os.path.join(run, "index.md")):
|
|
sys.exit(1)
|
|
res = subprocess.run([sys.executable, "-m", "okfwiki.audit", run, os.environ["V1_SOURCES"]], capture_output=True, text=True)
|
|
if res.returncode not in (0, 1):
|
|
sys.exit(1)
|
|
out = json.loads(res.stdout)
|
|
rule = out.get("link_rule")
|
|
if not isinstance(rule, dict):
|
|
sys.exit(1)
|
|
main, linked = rule.get("main_pages"), rule.get("linked")
|
|
if not isinstance(main, dict) or not isinstance(linked, dict) or not isinstance(main.get("of"), int) or main["of"] < 1:
|
|
sys.exit(1)
|
|
classes = ("link-misplaced", "broken-link", "main-page-missing", "main-page-duplicate", "main-page-invalid", "no-main-page-link")
|
|
bad = [f for f in out.get("findings", []) if f.get("check") in classes]
|
|
sys.exit(0 if main.get("pass") == main["of"] and linked.get("pass") == linked.get("of") and not bad else 1)'
|
|
check "$RUN auditor exits 0 and two runs give identical bytes" auditor_twice
|
|
check "$RUN view render and view check exit 0" view_check
|
|
check "$RUN view versions and view overlap report 0" view_versions_overlap
|
|
check "$RUN kost.json has a margin and cost summarize is within its cap" py '
|
|
import json, os, re, subprocess, sys
|
|
run = os.path.join(os.environ["V1_ROOT"], os.environ["RUN"])
|
|
kost = os.path.join(run, "kost.json")
|
|
if not os.path.isfile(kost) or "margin" not in json.load(open(kost)):
|
|
sys.exit(1)
|
|
res = subprocess.run([sys.executable, "-m", "okfwiki.cost", "summarize", "--kost", kost], capture_output=True, text=True)
|
|
lines = [m for m in (re.search(r"= ([0-9]+) of ([0-9]+)", x) for x in res.stdout.splitlines()) if m]
|
|
sys.exit(0 if res.returncode == 0 and lines and all(int(m.group(1)) <= int(m.group(2)) for m in lines) else 1)'
|
|
check "$RUN criterion 3: lesing.json says 20 of 20 with the draw sha256" py '
|
|
import json, os, re, sys
|
|
f = os.path.join(os.environ["V1_ROOT"], os.environ["RUN"], "lesing.json")
|
|
d = json.load(open(f)) if os.path.isfile(f) else {}
|
|
sys.exit(0 if d.get("answer") == "20 of 20" and re.fullmatch(r"[0-9a-f]{64}", str(d.get("sha256", ""))) else 1)'
|
|
check "$RUN criterion 2: okfwiki.claims score exits 0" criterion_2
|
|
check "$RUN Google viewer reads the bundle (tools/google_viewer_check.py)" google_viewer
|
|
check "$RUN every x- field has a measured reason (tools/x_fields_check.py)" x_fields
|
|
check "$RUN isolation breaks 0: each reader reads only its task file" py '
|
|
import json, os, sys
|
|
f = os.path.join(os.environ["V1_ROOT"], os.environ["RUN"], "isolation.json")
|
|
d = json.load(open(f)) if os.path.isfile(f) else {}
|
|
sys.exit(0 if d.get("breaks") == 0 else 1)'
|
|
check "$RUN lesing.json links: a links yes/no for each of the 20 pages" py '
|
|
import json, os, sys
|
|
f = os.path.join(os.environ["V1_ROOT"], os.environ["RUN"], "lesing.json")
|
|
pages = (json.load(open(f)) if os.path.isfile(f) else {}).get("pages")
|
|
sys.exit(0 if isinstance(pages, list) and len(pages) == 20 and all(isinstance(p, dict) and p.get("links") in ("yes", "no") for p in pages) else 1)'
|
|
check "$RUN criterion 4: both arms answered the same questions (K5); on samling1 the bundle arm is better, McNemar p < 0.05" py '
|
|
import json, math, os, sys
|
|
d = os.path.join(os.environ["V1_ROOT"], os.environ["RUN"], "svar")
|
|
arms = {}
|
|
for arm in ("filsok", "bundle"):
|
|
f = os.path.join(d, arm + ".json")
|
|
x = json.load(open(f)) if os.path.isfile(f) else {}
|
|
res = x.get("results") if isinstance(x, dict) else None
|
|
if x.get("schema") != "okf-wiki/answer-results/1" or x.get("arm") != arm or not isinstance(res, list) or not res:
|
|
sys.exit(1)
|
|
if not all(isinstance(r, dict) and isinstance(r.get("id"), str) and isinstance(r.get("correct"), bool) for r in res):
|
|
sys.exit(1)
|
|
arms[arm] = (x.get("set_sha256"), {r["id"]: r["correct"] for r in res})
|
|
(set_a, a), (set_b, b) = arms["filsok"], arms["bundle"]
|
|
if not set_a or set_a != set_b or set(a) != set(b):
|
|
sys.exit(1)
|
|
if os.environ["RUN"] != "samling1":
|
|
sys.exit(0)
|
|
only_a = sum(1 for i in a if a[i] and not b[i])
|
|
only_b = sum(1 for i in a if b[i] and not a[i])
|
|
n = only_a + only_b
|
|
p = min(1.0, 2 * sum(math.comb(n, k) for k in range(min(only_a, only_b) + 1)) / 2 ** n) if n else 1.0
|
|
sys.exit(0 if only_b > only_a and p < 0.05 else 1)'
|
|
done
|
|
check "change warning: the frozen copy of samling1 against a fresh fetch, marks shown in the view" change_warning
|
|
check "review: 0 BLOCKER (SC19)" py '
|
|
import os, re, sys
|
|
f = os.path.join(os.environ["V1_PROJECT"], "review.md")
|
|
text = open(f, encoding="utf-8").read() if os.path.isfile(f) else ""
|
|
front = re.match(r"---\n(.*?)\n---\n", text, re.S)
|
|
count = re.search(r"^- BLOCKER: ([0-9]+)$", text, re.M)
|
|
sys.exit(0 if front and re.search(r"^type: trekreview$", front.group(1), re.M) and count and count.group(1) == "0" else 1)'
|
|
exit "$FAILED"
|