#!/usr/bin/env bash # test-sc1-dogfood-log.sh — Verifies SC1 (operator-attested dogfood log) in REMEMBER.md # # Usage: # bash tests/test-sc1-dogfood-log.sh # missing block = WARN, exit 0 # bash tests/test-sc1-dogfood-log.sh --strict # missing block = FAIL, exit 1 # # The gate accepts EITHER of two block formats. It passes if at least ONE of # them is present AND valid; a second, malformed block downgrades to a WARN. # # v0.1 block — heading `## Dogfood log — v0.1 slides run`, five fields: # artifact_type: # refine_rounds: # final_prompt: # ``` # # ``` # shipped: yes (or `shipped: equivalent`) # comparison_to_unaided: # # v0.2 block (SC11, Phase 9 critique-iterate) — heading `## Dogfood log — v0.2 ...`, # four labelled sub-sections as `###` headings. Headings are matched on a # case-insensitive KEYWORD, not an exact label, so operator phrasing is free: # ### ...artifact... the exported HTML/screenshot reference. Must be a # file path — Claude Design artifacts have no URL. # ### ...critique... raw `design:critique` output excerpt; needs >=2 # non-blank lines (the findings list, not just a # verdict line). # ### ...re-prompt... the layer-targeted re-prompt from Phase 9c. # ### ...second pass... second-pass artifact result + one-line note. # # REMEMBER.md is gitignored — this evidence is local-only. The script # validates format only; the outcome judgement is operator-attested. set -euo pipefail LC_ALL=en_US.UTF-8 RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' PLUGIN_ROOT="$(cd "$(dirname "$0")/.." && pwd)" PASS=0 FAIL=0 WARN=0 pass() { printf "${GREEN} ✓ %s${NC}\n" "$1"; PASS=$((PASS + 1)); } fail() { printf "${RED} ✗ %s${NC}\n" "$1"; FAIL=$((FAIL + 1)); } warn() { printf "${YELLOW} ⚠ %s${NC}\n" "$1"; WARN=$((WARN + 1)); } STRICT=false for arg in "$@"; do case "$arg" in --strict) STRICT=true ;; esac done echo "=== test-sc1-dogfood-log ===" echo "Plugin root: $PLUGIN_ROOT" echo "Strict mode: $STRICT" echo "" REMEMBER_FILE="$PLUGIN_ROOT/REMEMBER.md" COVERAGE_FILE="$PLUGIN_ROOT/.coverage.md" # ------------------------------------------------------- # Deferred result recorder # # Each candidate block's checks are recorded, not printed, so the gate can # decide afterwards which block satisfies it. A recorded line is 'P|msg' or # 'F|msg'; replay() turns those into real pass()/fail() calls. # ------------------------------------------------------- LOG_DIR="$(mktemp -d)" trap 'rm -rf "$LOG_DIR"' EXIT CUR_LOG="$LOG_DIR/scratch" CUR_FAILS=0 rec_pass() { printf 'P|%s\n' "$1" >> "$CUR_LOG"; } rec_fail() { printf 'F|%s\n' "$1" >> "$CUR_LOG"; CUR_FAILS=$((CUR_FAILS + 1)); } replay() { local logfile="$1" [ -f "$logfile" ] || return 0 while IFS='|' read -r kind msg; do [ -z "$kind" ] && continue if [ "$kind" = "P" ]; then pass "$msg" else fail "$msg" fi done < "$logfile" } # ------------------------------------------------------- # Locate REMEMBER.md # ------------------------------------------------------- if [ ! -f "$REMEMBER_FILE" ]; then if $STRICT; then fail "REMEMBER.md missing (strict mode — required)" echo "" echo "=== Summary ===" printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN" exit 1 else warn "REMEMBER.md missing (advisory until operator dogfood step)" echo "" echo "=== Summary ===" printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN" exit 0 fi fi # ------------------------------------------------------- # Extract candidate blocks (heading → next H2 or EOF) # ------------------------------------------------------- V01_BLOCK="$(awk ' /^## Dogfood log — v0\.1 slides run$/ { capture = 1; next } capture && /^## / { exit } capture { print } ' "$REMEMBER_FILE")" V02_BLOCK="$(awk ' /^## Dogfood log — v0\.2/ { capture = 1; next } capture && /^## / { exit } capture { print } ' "$REMEMBER_FILE")" if [ -z "$V01_BLOCK" ] && [ -z "$V02_BLOCK" ]; then if $STRICT; then fail "REMEMBER.md missing dogfood block '## Dogfood log — v0.1 slides run' or '## Dogfood log — v0.2 ...' (strict mode — required)" echo "" echo "=== Summary ===" printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN" exit 1 else warn "REMEMBER.md missing dogfood block (advisory until operator dogfood step)" echo "" echo "=== Summary ===" printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN" exit 0 fi fi # ------------------------------------------------------- # v0.1 checks — logic unchanged from the original contract # ------------------------------------------------------- check_v01() { local BLOCK="$V01_BLOCK" rec_pass "found '## Dogfood log — v0.1 slides run' block" # Field 1: artifact_type — must match a preset name from .coverage.md local ARTIFACT_TYPE PRESETS FOUND preset ARTIFACT_TYPE="$(printf '%s\n' "$BLOCK" | awk -F': *' '/^artifact_type:/ { print $2; exit }' | tr -d '[:space:]')" if [ -z "$ARTIFACT_TYPE" ]; then rec_fail "artifact_type: field missing or empty" else if [ -f "$COVERAGE_FILE" ]; then PRESETS="$(awk -F'|' ' /^\| [a-z]/ { gsub(/^ +| +$/, "", $2); print $2 } ' "$COVERAGE_FILE")" FOUND=false while IFS= read -r preset; do [ -z "$preset" ] && continue if [ "$preset" = "$ARTIFACT_TYPE" ]; then FOUND=true break fi done < <(printf '%s\n' "$PRESETS") if $FOUND; then rec_pass "artifact_type='$ARTIFACT_TYPE' matches a preset in .coverage.md" else rec_fail "artifact_type='$ARTIFACT_TYPE' does not match any preset in .coverage.md" fi else rec_fail ".coverage.md missing — cannot validate artifact_type" fi fi # Field 2: refine_rounds — integer local REFINE_ROUNDS_LINE REFINE_ROUNDS_LINE="$(printf '%s\n' "$BLOCK" | grep -E '^refine_rounds:[[:space:]]*[0-9]+[[:space:]]*$' || true)" if [ -n "$REFINE_ROUNDS_LINE" ]; then rec_pass "refine_rounds: matches integer regex" else rec_fail "refine_rounds: missing or not an integer" fi # Field 3: final_prompt: followed by non-empty fenced code block local HAS_FINAL_PROMPT FENCE_AFTER HAS_FINAL_PROMPT="$(printf '%s\n' "$BLOCK" | grep -c '^final_prompt:' || true)" if [ "$HAS_FINAL_PROMPT" -ge 1 ]; then FENCE_AFTER="$(awk ' /^final_prompt:/ { found = 1; next } found && /^```/ { fence_open = !fence_open; if (fence_open) { in_fence = 1 } else { exit } } found && in_fence && fence_open && /./ { content_lines++ } END { print content_lines + 0 } ' <<<"$BLOCK")" if [ -z "$FENCE_AFTER" ]; then FENCE_AFTER=0; fi if [ "$FENCE_AFTER" -ge 1 ]; then rec_pass "final_prompt: followed by non-empty fenced code block ($FENCE_AFTER content line(s))" else rec_fail "final_prompt: not followed by a non-empty fenced code block" fi else rec_fail "final_prompt: field missing" fi # Field 4: shipped — yes or equivalent local SHIPPED_LINE SHIPPED_LINE="$(printf '%s\n' "$BLOCK" | grep -E '^shipped:[[:space:]]*(yes|equivalent)[[:space:]]*$' || true)" if [ -n "$SHIPPED_LINE" ]; then rec_pass "shipped: matches 'yes' or 'equivalent'" else rec_fail "shipped: missing or not 'yes'/'equivalent'" fi # Field 5: comparison_to_unaided — non-empty sentence >=10 chars ending with . local COMP_LINE COMP_TRIMMED COMP_LEN COMP_LINE="$(printf '%s\n' "$BLOCK" | awk -F': *' '/^comparison_to_unaided:/ { for (i=2;i<=NF;i++) printf "%s%s", $i, (i=10)" elif [ "${COMP_TRIMMED: -1}" != "." ]; then rec_fail "comparison_to_unaided: does not end with '.'" else rec_pass "comparison_to_unaided: non-empty, $COMP_LEN chars, ends with '.'" fi } # ------------------------------------------------------- # v0.2 checks — SC11 Phase 9 critique-iterate block # ------------------------------------------------------- # Body of the `###` sub-section whose heading matches a lowercase keyword regex. v02_section() { printf '%s\n' "$V02_BLOCK" | awk -v re="$1" ' /^###+ / { if (tolower($0) ~ re) { capture = 1; next } if (capture) { exit } next } capture { print } ' } # $1 = human label, $2 = lowercase keyword regex, $3 = minimum non-blank lines check_v02_section() { local label="$1" re="$2" minlines="$3" local heading body nonblank heading="$(printf '%s\n' "$V02_BLOCK" | grep -E '^###+ ' | tr '[:upper:]' '[:lower:]' | grep -E "$re" || true)" if [ -z "$heading" ]; then rec_fail "$label: no '###' sub-section heading matching /$re/" return 0 fi body="$(v02_section "$re" || true)" nonblank="$(printf '%s\n' "$body" | grep -c '[^[:space:]]' || true)" if [ -z "$nonblank" ]; then nonblank=0; fi if [ "$nonblank" -ge "$minlines" ]; then rec_pass "$label: sub-section present with $nonblank non-blank line(s) (need >=$minlines)" else rec_fail "$label: sub-section has $nonblank non-blank line(s) (need >=$minlines)" fi } check_v02() { rec_pass "found '## Dogfood log — v0.2' block" # Sub-section 1: artifact reference (exported file path or screenshot) check_v02_section "artifact_ref" 'artifact' 1 # Artifact reference must be a path, not a URL — Claude Design artifacts # have no URL, so a URL-only reference is unverifiable evidence. local art_body art_nonblank art_urls art_body="$(v02_section 'artifact' || true)" art_nonblank="$(printf '%s\n' "$art_body" | grep -c '[^[:space:]]' || true)" if [ -z "$art_nonblank" ]; then art_nonblank=0; fi if [ "$art_nonblank" -ge 1 ]; then art_urls="$(printf '%s\n' "$art_body" | grep -c 'https\?://' || true)" if [ -z "$art_urls" ]; then art_urls=0; fi if [ "$art_urls" -ge "$art_nonblank" ]; then rec_fail "artifact_ref: every line is a URL — SC11 requires an exported file path or screenshot (Claude Design artifacts have no URL)" else rec_pass "artifact_ref: carries a non-URL file reference" fi fi # Sub-section 2: raw design:critique output excerpt — findings list, not a # bare verdict line, hence >=2 non-blank lines check_v02_section "critique_output" 'critique' 2 # Sub-section 3: layer-targeted re-prompt from Phase 9c check_v02_section "layer_targeted_reprompt" 're-?prompt' 1 # Sub-section 4: second-pass artifact result + one-line note check_v02_section "second_pass_result" 'second[ -]?pass' 1 } # ------------------------------------------------------- # Run whichever blocks are present, then decide the gate # ------------------------------------------------------- V01_FAILS=-1 V02_FAILS=-1 if [ -n "$V01_BLOCK" ]; then CUR_LOG="$LOG_DIR/v01"; : > "$CUR_LOG"; CUR_FAILS=0 check_v01 V01_FAILS=$CUR_FAILS fi if [ -n "$V02_BLOCK" ]; then CUR_LOG="$LOG_DIR/v02"; : > "$CUR_LOG"; CUR_FAILS=0 check_v02 V02_FAILS=$CUR_FAILS fi if [ -n "$V01_BLOCK" ] && [ "$V01_FAILS" -eq 0 ]; then replay "$LOG_DIR/v01" if [ -n "$V02_BLOCK" ] && [ "$V02_FAILS" -gt 0 ]; then warn "v0.2 block present with $V02_FAILS format issue(s); gate satisfied by the valid v0.1 block" elif [ -n "$V02_BLOCK" ]; then replay "$LOG_DIR/v02" fi elif [ -n "$V02_BLOCK" ] && [ "$V02_FAILS" -eq 0 ]; then replay "$LOG_DIR/v02" if [ -n "$V01_BLOCK" ]; then warn "v0.1 block present with $V01_FAILS format issue(s); gate satisfied by the valid v0.2 block" fi else if [ -n "$V01_BLOCK" ]; then replay "$LOG_DIR/v01"; fi if [ -n "$V02_BLOCK" ]; then replay "$LOG_DIR/v02"; fi fi # ------------------------------------------------------- # Summary # ------------------------------------------------------- echo "" echo "=== Summary ===" printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN" if [ "$FAIL" -gt 0 ]; then exit 1 fi exit 0