test(claude-design): add SC4 coverage test + accept v0.2 dogfood block in SC1
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012LeEowUoirsMeXBpJGps6h
This commit is contained in:
parent
7d604309ae
commit
e9c19a00b4
2 changed files with 389 additions and 78 deletions
|
|
@ -5,9 +5,10 @@
|
||||||
# bash tests/test-sc1-dogfood-log.sh # missing block = WARN, exit 0
|
# bash tests/test-sc1-dogfood-log.sh # missing block = WARN, exit 0
|
||||||
# bash tests/test-sc1-dogfood-log.sh --strict # missing block = FAIL, exit 1
|
# bash tests/test-sc1-dogfood-log.sh --strict # missing block = FAIL, exit 1
|
||||||
#
|
#
|
||||||
# Expects in REMEMBER.md (plugin root, gitignored):
|
# The gate accepts EITHER of two block formats. It passes if at least ONE of
|
||||||
# - A fenced section with heading `## Dogfood log — v0.1 slides run`
|
# them is present AND valid; a second, malformed block downgrades to a WARN.
|
||||||
# - Five mechanically-checkable fields inside the section:
|
#
|
||||||
|
# v0.1 block — heading `## Dogfood log — v0.1 slides run`, five fields:
|
||||||
# artifact_type: <preset-name from .coverage.md>
|
# artifact_type: <preset-name from .coverage.md>
|
||||||
# refine_rounds: <integer>
|
# refine_rounds: <integer>
|
||||||
# final_prompt:
|
# final_prompt:
|
||||||
|
|
@ -17,6 +18,17 @@
|
||||||
# shipped: yes (or `shipped: equivalent`)
|
# shipped: yes (or `shipped: equivalent`)
|
||||||
# comparison_to_unaided: <one sentence ending with .>
|
# comparison_to_unaided: <one sentence ending with .>
|
||||||
#
|
#
|
||||||
|
# v0.2 block (SC11, Phase 9 critique-iterate) — heading `## Dogfood log — v0.2 ...`,
|
||||||
|
# four labelled sub-sections as `###` headings. Headings are matched on a
|
||||||
|
# case-insensitive KEYWORD, not an exact label, so operator phrasing is free:
|
||||||
|
# ### ...artifact... the exported HTML/screenshot reference. Must be a
|
||||||
|
# file path — Claude Design artifacts have no URL.
|
||||||
|
# ### ...critique... raw `design:critique` output excerpt; needs >=2
|
||||||
|
# non-blank lines (the findings list, not just a
|
||||||
|
# verdict line).
|
||||||
|
# ### ...re-prompt... the layer-targeted re-prompt from Phase 9c.
|
||||||
|
# ### ...second pass... second-pass artifact result + one-line note.
|
||||||
|
#
|
||||||
# REMEMBER.md is gitignored — this evidence is local-only. The script
|
# REMEMBER.md is gitignored — this evidence is local-only. The script
|
||||||
# validates format only; the outcome judgement is operator-attested.
|
# validates format only; the outcome judgement is operator-attested.
|
||||||
|
|
||||||
|
|
@ -52,6 +64,35 @@ echo ""
|
||||||
REMEMBER_FILE="$PLUGIN_ROOT/REMEMBER.md"
|
REMEMBER_FILE="$PLUGIN_ROOT/REMEMBER.md"
|
||||||
COVERAGE_FILE="$PLUGIN_ROOT/.coverage.md"
|
COVERAGE_FILE="$PLUGIN_ROOT/.coverage.md"
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# Deferred result recorder
|
||||||
|
#
|
||||||
|
# Each candidate block's checks are recorded, not printed, so the gate can
|
||||||
|
# decide afterwards which block satisfies it. A recorded line is 'P|msg' or
|
||||||
|
# 'F|msg'; replay() turns those into real pass()/fail() calls.
|
||||||
|
# -------------------------------------------------------
|
||||||
|
LOG_DIR="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$LOG_DIR"' EXIT
|
||||||
|
|
||||||
|
CUR_LOG="$LOG_DIR/scratch"
|
||||||
|
CUR_FAILS=0
|
||||||
|
|
||||||
|
rec_pass() { printf 'P|%s\n' "$1" >> "$CUR_LOG"; }
|
||||||
|
rec_fail() { printf 'F|%s\n' "$1" >> "$CUR_LOG"; CUR_FAILS=$((CUR_FAILS + 1)); }
|
||||||
|
|
||||||
|
replay() {
|
||||||
|
local logfile="$1"
|
||||||
|
[ -f "$logfile" ] || return 0
|
||||||
|
while IFS='|' read -r kind msg; do
|
||||||
|
[ -z "$kind" ] && continue
|
||||||
|
if [ "$kind" = "P" ]; then
|
||||||
|
pass "$msg"
|
||||||
|
else
|
||||||
|
fail "$msg"
|
||||||
|
fi
|
||||||
|
done < "$logfile"
|
||||||
|
}
|
||||||
|
|
||||||
# -------------------------------------------------------
|
# -------------------------------------------------------
|
||||||
# Locate REMEMBER.md
|
# Locate REMEMBER.md
|
||||||
# -------------------------------------------------------
|
# -------------------------------------------------------
|
||||||
|
|
@ -72,17 +113,23 @@ if [ ! -f "$REMEMBER_FILE" ]; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# -------------------------------------------------------
|
# -------------------------------------------------------
|
||||||
# Extract fenced block between dogfood heading and next H2 (or EOF)
|
# Extract candidate blocks (heading → next H2 or EOF)
|
||||||
# -------------------------------------------------------
|
# -------------------------------------------------------
|
||||||
BLOCK="$(awk '
|
V01_BLOCK="$(awk '
|
||||||
/^## Dogfood log — v0\.1 slides run$/ { capture = 1; next }
|
/^## Dogfood log — v0\.1 slides run$/ { capture = 1; next }
|
||||||
capture && /^## / { exit }
|
capture && /^## / { exit }
|
||||||
capture { print }
|
capture { print }
|
||||||
' "$REMEMBER_FILE")"
|
' "$REMEMBER_FILE")"
|
||||||
|
|
||||||
if [ -z "$BLOCK" ]; then
|
V02_BLOCK="$(awk '
|
||||||
|
/^## Dogfood log — v0\.2/ { capture = 1; next }
|
||||||
|
capture && /^## / { exit }
|
||||||
|
capture { print }
|
||||||
|
' "$REMEMBER_FILE")"
|
||||||
|
|
||||||
|
if [ -z "$V01_BLOCK" ] && [ -z "$V02_BLOCK" ]; then
|
||||||
if $STRICT; then
|
if $STRICT; then
|
||||||
fail "REMEMBER.md missing dogfood block '## Dogfood log — v0.1 slides run' (strict mode — required)"
|
fail "REMEMBER.md missing dogfood block '## Dogfood log — v0.1 slides run' or '## Dogfood log — v0.2 ...' (strict mode — required)"
|
||||||
echo ""
|
echo ""
|
||||||
echo "=== Summary ==="
|
echo "=== Summary ==="
|
||||||
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||||
|
|
@ -96,98 +143,204 @@ if [ -z "$BLOCK" ]; then
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
pass "found '## Dogfood log — v0.1 slides run' block"
|
|
||||||
|
|
||||||
# -------------------------------------------------------
|
# -------------------------------------------------------
|
||||||
# Field 1: artifact_type — must match a preset name from .coverage.md
|
# v0.1 checks — logic unchanged from the original contract
|
||||||
# -------------------------------------------------------
|
# -------------------------------------------------------
|
||||||
ARTIFACT_TYPE="$(printf '%s\n' "$BLOCK" | awk -F': *' '/^artifact_type:/ { print $2; exit }' | tr -d '[:space:]')"
|
check_v01() {
|
||||||
|
local BLOCK="$V01_BLOCK"
|
||||||
|
|
||||||
if [ -z "$ARTIFACT_TYPE" ]; then
|
rec_pass "found '## Dogfood log — v0.1 slides run' block"
|
||||||
fail "artifact_type: field missing or empty"
|
|
||||||
else
|
|
||||||
# extract preset names from .coverage.md table column 1
|
|
||||||
if [ -f "$COVERAGE_FILE" ]; then
|
|
||||||
PRESETS="$(awk -F'|' '
|
|
||||||
/^\| [a-z]/ { gsub(/^ +| +$/, "", $2); print $2 }
|
|
||||||
' "$COVERAGE_FILE")"
|
|
||||||
|
|
||||||
FOUND=false
|
# Field 1: artifact_type — must match a preset name from .coverage.md
|
||||||
while IFS= read -r preset; do
|
local ARTIFACT_TYPE PRESETS FOUND preset
|
||||||
[ -z "$preset" ] && continue
|
ARTIFACT_TYPE="$(printf '%s\n' "$BLOCK" | awk -F': *' '/^artifact_type:/ { print $2; exit }' | tr -d '[:space:]')"
|
||||||
if [ "$preset" = "$ARTIFACT_TYPE" ]; then
|
|
||||||
FOUND=true
|
if [ -z "$ARTIFACT_TYPE" ]; then
|
||||||
break
|
rec_fail "artifact_type: field missing or empty"
|
||||||
|
else
|
||||||
|
if [ -f "$COVERAGE_FILE" ]; then
|
||||||
|
PRESETS="$(awk -F'|' '
|
||||||
|
/^\| [a-z]/ { gsub(/^ +| +$/, "", $2); print $2 }
|
||||||
|
' "$COVERAGE_FILE")"
|
||||||
|
|
||||||
|
FOUND=false
|
||||||
|
while IFS= read -r preset; do
|
||||||
|
[ -z "$preset" ] && continue
|
||||||
|
if [ "$preset" = "$ARTIFACT_TYPE" ]; then
|
||||||
|
FOUND=true
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done < <(printf '%s\n' "$PRESETS")
|
||||||
|
|
||||||
|
if $FOUND; then
|
||||||
|
rec_pass "artifact_type='$ARTIFACT_TYPE' matches a preset in .coverage.md"
|
||||||
|
else
|
||||||
|
rec_fail "artifact_type='$ARTIFACT_TYPE' does not match any preset in .coverage.md"
|
||||||
fi
|
fi
|
||||||
done < <(printf '%s\n' "$PRESETS")
|
|
||||||
|
|
||||||
if $FOUND; then
|
|
||||||
pass "artifact_type='$ARTIFACT_TYPE' matches a preset in .coverage.md"
|
|
||||||
else
|
else
|
||||||
fail "artifact_type='$ARTIFACT_TYPE' does not match any preset in .coverage.md"
|
rec_fail ".coverage.md missing — cannot validate artifact_type"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Field 2: refine_rounds — integer
|
||||||
|
local REFINE_ROUNDS_LINE
|
||||||
|
REFINE_ROUNDS_LINE="$(printf '%s\n' "$BLOCK" | grep -E '^refine_rounds:[[:space:]]*[0-9]+[[:space:]]*$' || true)"
|
||||||
|
if [ -n "$REFINE_ROUNDS_LINE" ]; then
|
||||||
|
rec_pass "refine_rounds: matches integer regex"
|
||||||
|
else
|
||||||
|
rec_fail "refine_rounds: missing or not an integer"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Field 3: final_prompt: followed by non-empty fenced code block
|
||||||
|
local HAS_FINAL_PROMPT FENCE_AFTER
|
||||||
|
HAS_FINAL_PROMPT="$(printf '%s\n' "$BLOCK" | grep -c '^final_prompt:' || true)"
|
||||||
|
if [ "$HAS_FINAL_PROMPT" -ge 1 ]; then
|
||||||
|
FENCE_AFTER="$(awk '
|
||||||
|
/^final_prompt:/ { found = 1; next }
|
||||||
|
found && /^```/ { fence_open = !fence_open; if (fence_open) { in_fence = 1 } else { exit } }
|
||||||
|
found && in_fence && fence_open && /./ { content_lines++ }
|
||||||
|
END { print content_lines + 0 }
|
||||||
|
' <<<"$BLOCK")"
|
||||||
|
if [ -z "$FENCE_AFTER" ]; then FENCE_AFTER=0; fi
|
||||||
|
if [ "$FENCE_AFTER" -ge 1 ]; then
|
||||||
|
rec_pass "final_prompt: followed by non-empty fenced code block ($FENCE_AFTER content line(s))"
|
||||||
|
else
|
||||||
|
rec_fail "final_prompt: not followed by a non-empty fenced code block"
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
fail ".coverage.md missing — cannot validate artifact_type"
|
rec_fail "final_prompt: field missing"
|
||||||
fi
|
fi
|
||||||
fi
|
|
||||||
|
|
||||||
# -------------------------------------------------------
|
# Field 4: shipped — yes or equivalent
|
||||||
# Field 2: refine_rounds — integer
|
local SHIPPED_LINE
|
||||||
# -------------------------------------------------------
|
SHIPPED_LINE="$(printf '%s\n' "$BLOCK" | grep -E '^shipped:[[:space:]]*(yes|equivalent)[[:space:]]*$' || true)"
|
||||||
REFINE_ROUNDS_LINE="$(printf '%s\n' "$BLOCK" | grep -E '^refine_rounds:[[:space:]]*[0-9]+[[:space:]]*$' || true)"
|
if [ -n "$SHIPPED_LINE" ]; then
|
||||||
if [ -n "$REFINE_ROUNDS_LINE" ]; then
|
rec_pass "shipped: matches 'yes' or 'equivalent'"
|
||||||
pass "refine_rounds: matches integer regex"
|
|
||||||
else
|
|
||||||
fail "refine_rounds: missing or not an integer"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# -------------------------------------------------------
|
|
||||||
# Field 3: final_prompt: followed by non-empty fenced code block
|
|
||||||
# -------------------------------------------------------
|
|
||||||
HAS_FINAL_PROMPT="$(printf '%s\n' "$BLOCK" | grep -c '^final_prompt:' || true)"
|
|
||||||
if [ "$HAS_FINAL_PROMPT" -ge 1 ]; then
|
|
||||||
# check that a fenced code block (```) appears after final_prompt:
|
|
||||||
FENCE_AFTER="$(awk '
|
|
||||||
/^final_prompt:/ { found = 1; next }
|
|
||||||
found && /^```/ { fence_open = !fence_open; if (fence_open) { in_fence = 1 } else { exit } }
|
|
||||||
found && in_fence && fence_open && /./ { content_lines++ }
|
|
||||||
END { print content_lines + 0 }
|
|
||||||
' <<<"$BLOCK")"
|
|
||||||
if [ -z "$FENCE_AFTER" ]; then FENCE_AFTER=0; fi
|
|
||||||
if [ "$FENCE_AFTER" -ge 1 ]; then
|
|
||||||
pass "final_prompt: followed by non-empty fenced code block ($FENCE_AFTER content line(s))"
|
|
||||||
else
|
else
|
||||||
fail "final_prompt: not followed by a non-empty fenced code block"
|
rec_fail "shipped: missing or not 'yes'/'equivalent'"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Field 5: comparison_to_unaided — non-empty sentence >=10 chars ending with .
|
||||||
|
local COMP_LINE COMP_TRIMMED COMP_LEN
|
||||||
|
COMP_LINE="$(printf '%s\n' "$BLOCK" | awk -F': *' '/^comparison_to_unaided:/ { for (i=2;i<=NF;i++) printf "%s%s", $i, (i<NF?": ":""); print ""; exit }')"
|
||||||
|
COMP_TRIMMED="$(printf '%s' "$COMP_LINE" | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//')"
|
||||||
|
COMP_LEN="${#COMP_TRIMMED}"
|
||||||
|
|
||||||
|
if [ -z "$COMP_TRIMMED" ]; then
|
||||||
|
rec_fail "comparison_to_unaided: field missing or empty"
|
||||||
|
elif [ "$COMP_LEN" -lt 10 ]; then
|
||||||
|
rec_fail "comparison_to_unaided: too short ($COMP_LEN chars; need >=10)"
|
||||||
|
elif [ "${COMP_TRIMMED: -1}" != "." ]; then
|
||||||
|
rec_fail "comparison_to_unaided: does not end with '.'"
|
||||||
|
else
|
||||||
|
rec_pass "comparison_to_unaided: non-empty, $COMP_LEN chars, ends with '.'"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# v0.2 checks — SC11 Phase 9 critique-iterate block
|
||||||
|
# -------------------------------------------------------
|
||||||
|
|
||||||
|
# Body of the `###` sub-section whose heading matches a lowercase keyword regex.
|
||||||
|
v02_section() {
|
||||||
|
printf '%s\n' "$V02_BLOCK" | awk -v re="$1" '
|
||||||
|
/^###+ / {
|
||||||
|
if (tolower($0) ~ re) { capture = 1; next }
|
||||||
|
if (capture) { exit }
|
||||||
|
next
|
||||||
|
}
|
||||||
|
capture { print }
|
||||||
|
'
|
||||||
|
}
|
||||||
|
|
||||||
|
# $1 = human label, $2 = lowercase keyword regex, $3 = minimum non-blank lines
|
||||||
|
check_v02_section() {
|
||||||
|
local label="$1" re="$2" minlines="$3"
|
||||||
|
local heading body nonblank
|
||||||
|
|
||||||
|
heading="$(printf '%s\n' "$V02_BLOCK" | grep -E '^###+ ' | tr '[:upper:]' '[:lower:]' | grep -E "$re" || true)"
|
||||||
|
if [ -z "$heading" ]; then
|
||||||
|
rec_fail "$label: no '###' sub-section heading matching /$re/"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
body="$(v02_section "$re" || true)"
|
||||||
|
nonblank="$(printf '%s\n' "$body" | grep -c '[^[:space:]]' || true)"
|
||||||
|
if [ -z "$nonblank" ]; then nonblank=0; fi
|
||||||
|
|
||||||
|
if [ "$nonblank" -ge "$minlines" ]; then
|
||||||
|
rec_pass "$label: sub-section present with $nonblank non-blank line(s) (need >=$minlines)"
|
||||||
|
else
|
||||||
|
rec_fail "$label: sub-section has $nonblank non-blank line(s) (need >=$minlines)"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_v02() {
|
||||||
|
rec_pass "found '## Dogfood log — v0.2' block"
|
||||||
|
|
||||||
|
# Sub-section 1: artifact reference (exported file path or screenshot)
|
||||||
|
check_v02_section "artifact_ref" 'artifact' 1
|
||||||
|
|
||||||
|
# Artifact reference must be a path, not a URL — Claude Design artifacts
|
||||||
|
# have no URL, so a URL-only reference is unverifiable evidence.
|
||||||
|
local art_body art_nonblank art_urls
|
||||||
|
art_body="$(v02_section 'artifact' || true)"
|
||||||
|
art_nonblank="$(printf '%s\n' "$art_body" | grep -c '[^[:space:]]' || true)"
|
||||||
|
if [ -z "$art_nonblank" ]; then art_nonblank=0; fi
|
||||||
|
if [ "$art_nonblank" -ge 1 ]; then
|
||||||
|
art_urls="$(printf '%s\n' "$art_body" | grep -c 'https\?://' || true)"
|
||||||
|
if [ -z "$art_urls" ]; then art_urls=0; fi
|
||||||
|
if [ "$art_urls" -ge "$art_nonblank" ]; then
|
||||||
|
rec_fail "artifact_ref: every line is a URL — SC11 requires an exported file path or screenshot (Claude Design artifacts have no URL)"
|
||||||
|
else
|
||||||
|
rec_pass "artifact_ref: carries a non-URL file reference"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Sub-section 2: raw design:critique output excerpt — findings list, not a
|
||||||
|
# bare verdict line, hence >=2 non-blank lines
|
||||||
|
check_v02_section "critique_output" 'critique' 2
|
||||||
|
|
||||||
|
# Sub-section 3: layer-targeted re-prompt from Phase 9c
|
||||||
|
check_v02_section "layer_targeted_reprompt" 're-?prompt' 1
|
||||||
|
|
||||||
|
# Sub-section 4: second-pass artifact result + one-line note
|
||||||
|
check_v02_section "second_pass_result" 'second[ -]?pass' 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# Run whichever blocks are present, then decide the gate
|
||||||
|
# -------------------------------------------------------
|
||||||
|
V01_FAILS=-1
|
||||||
|
V02_FAILS=-1
|
||||||
|
|
||||||
|
if [ -n "$V01_BLOCK" ]; then
|
||||||
|
CUR_LOG="$LOG_DIR/v01"; : > "$CUR_LOG"; CUR_FAILS=0
|
||||||
|
check_v01
|
||||||
|
V01_FAILS=$CUR_FAILS
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "$V02_BLOCK" ]; then
|
||||||
|
CUR_LOG="$LOG_DIR/v02"; : > "$CUR_LOG"; CUR_FAILS=0
|
||||||
|
check_v02
|
||||||
|
V02_FAILS=$CUR_FAILS
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "$V01_BLOCK" ] && [ "$V01_FAILS" -eq 0 ]; then
|
||||||
|
replay "$LOG_DIR/v01"
|
||||||
|
if [ -n "$V02_BLOCK" ] && [ "$V02_FAILS" -gt 0 ]; then
|
||||||
|
warn "v0.2 block present with $V02_FAILS format issue(s); gate satisfied by the valid v0.1 block"
|
||||||
|
elif [ -n "$V02_BLOCK" ]; then
|
||||||
|
replay "$LOG_DIR/v02"
|
||||||
|
fi
|
||||||
|
elif [ -n "$V02_BLOCK" ] && [ "$V02_FAILS" -eq 0 ]; then
|
||||||
|
replay "$LOG_DIR/v02"
|
||||||
|
if [ -n "$V01_BLOCK" ]; then
|
||||||
|
warn "v0.1 block present with $V01_FAILS format issue(s); gate satisfied by the valid v0.2 block"
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
fail "final_prompt: field missing"
|
if [ -n "$V01_BLOCK" ]; then replay "$LOG_DIR/v01"; fi
|
||||||
fi
|
if [ -n "$V02_BLOCK" ]; then replay "$LOG_DIR/v02"; fi
|
||||||
|
|
||||||
# -------------------------------------------------------
|
|
||||||
# Field 4: shipped — yes or equivalent
|
|
||||||
# -------------------------------------------------------
|
|
||||||
SHIPPED_LINE="$(printf '%s\n' "$BLOCK" | grep -E '^shipped:[[:space:]]*(yes|equivalent)[[:space:]]*$' || true)"
|
|
||||||
if [ -n "$SHIPPED_LINE" ]; then
|
|
||||||
pass "shipped: matches 'yes' or 'equivalent'"
|
|
||||||
else
|
|
||||||
fail "shipped: missing or not 'yes'/'equivalent'"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# -------------------------------------------------------
|
|
||||||
# Field 5: comparison_to_unaided — non-empty sentence >=10 chars ending with .
|
|
||||||
# -------------------------------------------------------
|
|
||||||
COMP_LINE="$(printf '%s\n' "$BLOCK" | awk -F': *' '/^comparison_to_unaided:/ { for (i=2;i<=NF;i++) printf "%s%s", $i, (i<NF?": ":""); print ""; exit }')"
|
|
||||||
COMP_TRIMMED="$(printf '%s' "$COMP_LINE" | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//')"
|
|
||||||
COMP_LEN="${#COMP_TRIMMED}"
|
|
||||||
|
|
||||||
if [ -z "$COMP_TRIMMED" ]; then
|
|
||||||
fail "comparison_to_unaided: field missing or empty"
|
|
||||||
elif [ "$COMP_LEN" -lt 10 ]; then
|
|
||||||
fail "comparison_to_unaided: too short ($COMP_LEN chars; need >=10)"
|
|
||||||
elif [ "${COMP_TRIMMED: -1}" != "." ]; then
|
|
||||||
fail "comparison_to_unaided: does not end with '.'"
|
|
||||||
else
|
|
||||||
pass "comparison_to_unaided: non-empty, $COMP_LEN chars, ends with '.'"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# -------------------------------------------------------
|
# -------------------------------------------------------
|
||||||
|
|
|
||||||
158
tests/test-sc4-phase9-coverage.sh
Executable file
158
tests/test-sc4-phase9-coverage.sh
Executable file
|
|
@ -0,0 +1,158 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# test-sc4-phase9-coverage.sh — Verifies SC4 (Phase 9 critique-iterate coverage)
|
||||||
|
#
|
||||||
|
# Four checks:
|
||||||
|
# (1) references/05-critique-iterate.md exists and carries the four
|
||||||
|
# sub-phase headings '## 9a'..'## 9d'.
|
||||||
|
# (2) SKILL.md carries the '## Phase 9 — Critique-iterate' section heading,
|
||||||
|
# at least 5 'Phase 9' occurrences, and '### 9a'..'### 9d'.
|
||||||
|
# (3) Every references/presets/*.md carries a critique-iterate hint block.
|
||||||
|
# ANCHOR NOTE: the hint block's heading letter is PER FILE — four presets
|
||||||
|
# end at '## (e)' and got '## (f) Critique-iterate hint'; four end at
|
||||||
|
# '## (f)' and got '## (g) Critique-iterate hint'. The canonical
|
||||||
|
# machine-checkable anchor is therefore the LETTER-FREE substring
|
||||||
|
# 'Critique-iterate hint'. Never anchor on the letter: a test grepping
|
||||||
|
# '## (g)' would fail on four files, and the "fix" would be to break the
|
||||||
|
# letter sequence in shipped content.
|
||||||
|
# (4) references/05-critique-iterate.md carries >=1 Anthropic-domain citation.
|
||||||
|
#
|
||||||
|
# Anthropic-domain URL regex (same as test-sc3-citations.sh):
|
||||||
|
# https?://(docs\.anthropic\.com|anthropic\.com|github\.com/anthropics
|
||||||
|
# |claude\.com|support\.claude\.com|platform\.claude\.com)
|
||||||
|
#
|
||||||
|
# Usage: bash tests/test-sc4-phase9-coverage.sh
|
||||||
|
# Exit codes: 0 = pass; 1 = at least one FAIL
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
LC_ALL=en_US.UTF-8
|
||||||
|
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
YELLOW='\033[1;33m'
|
||||||
|
NC='\033[0m'
|
||||||
|
|
||||||
|
PLUGIN_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
PASS=0
|
||||||
|
FAIL=0
|
||||||
|
WARN=0
|
||||||
|
|
||||||
|
pass() { printf "${GREEN} ✓ %s${NC}\n" "$1"; PASS=$((PASS + 1)); }
|
||||||
|
fail() { printf "${RED} ✗ %s${NC}\n" "$1"; FAIL=$((FAIL + 1)); }
|
||||||
|
warn() { printf "${YELLOW} ⚠ %s${NC}\n" "$1"; WARN=$((WARN + 1)); }
|
||||||
|
|
||||||
|
SKILL_DIR="$PLUGIN_ROOT/skills/claude-design-facilitator"
|
||||||
|
CRITIQUE_REF="$SKILL_DIR/references/05-critique-iterate.md"
|
||||||
|
SKILL_FILE="$SKILL_DIR/SKILL.md"
|
||||||
|
PRESET_DIR="$SKILL_DIR/references/presets"
|
||||||
|
|
||||||
|
ANTHROPIC_REGEX='https?://(docs\.anthropic\.com|anthropic\.com|github\.com/anthropics|claude\.com|support\.claude\.com|platform\.claude\.com)'
|
||||||
|
|
||||||
|
echo "=== test-sc4-phase9-coverage ==="
|
||||||
|
echo "Plugin root: $PLUGIN_ROOT"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# (1) 05-critique-iterate.md exists with 9a..9d sub-phases
|
||||||
|
# -------------------------------------------------------
|
||||||
|
echo "--- (1) shared critique-iterate reference ---"
|
||||||
|
|
||||||
|
if [ ! -f "$CRITIQUE_REF" ]; then
|
||||||
|
fail "references/05-critique-iterate.md missing"
|
||||||
|
else
|
||||||
|
pass "references/05-critique-iterate.md exists"
|
||||||
|
for sub in 9a 9b 9c 9d; do
|
||||||
|
if grep -q "^## $sub" "$CRITIQUE_REF"; then
|
||||||
|
pass "05-critique-iterate.md: '## $sub' heading present"
|
||||||
|
else
|
||||||
|
fail "05-critique-iterate.md: '## $sub' heading missing"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# (2) SKILL.md Phase 9 section
|
||||||
|
# -------------------------------------------------------
|
||||||
|
echo "--- (2) SKILL.md Phase 9 section ---"
|
||||||
|
|
||||||
|
if [ ! -f "$SKILL_FILE" ]; then
|
||||||
|
fail "SKILL.md missing"
|
||||||
|
else
|
||||||
|
if grep -qF '## Phase 9 — Critique-iterate' "$SKILL_FILE"; then
|
||||||
|
pass "SKILL.md: '## Phase 9 — Critique-iterate' heading present"
|
||||||
|
else
|
||||||
|
fail "SKILL.md: '## Phase 9 — Critique-iterate' heading missing"
|
||||||
|
fi
|
||||||
|
|
||||||
|
P9_COUNT="$(grep -oF 'Phase 9' "$SKILL_FILE" | wc -l | tr -d '[:space:]')"
|
||||||
|
if [ -z "$P9_COUNT" ]; then P9_COUNT=0; fi
|
||||||
|
if [ "$P9_COUNT" -ge 5 ]; then
|
||||||
|
pass "SKILL.md: $P9_COUNT 'Phase 9' occurrence(s) (need >=5)"
|
||||||
|
else
|
||||||
|
fail "SKILL.md: only $P9_COUNT 'Phase 9' occurrence(s) (need >=5)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
for sub in 9a 9b 9c 9d; do
|
||||||
|
if grep -q "^### $sub" "$SKILL_FILE"; then
|
||||||
|
pass "SKILL.md: '### $sub' heading present"
|
||||||
|
else
|
||||||
|
fail "SKILL.md: '### $sub' heading missing"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# (3) Every preset carries a critique-iterate hint block
|
||||||
|
# Anchored on the LETTER-FREE substring (see header note).
|
||||||
|
# -------------------------------------------------------
|
||||||
|
echo "--- (3) per-preset critique-iterate hint blocks ---"
|
||||||
|
|
||||||
|
if [ ! -d "$PRESET_DIR" ]; then
|
||||||
|
fail "references/presets/ directory missing"
|
||||||
|
else
|
||||||
|
PRESET_COUNT=0
|
||||||
|
while IFS= read -r preset_file; do
|
||||||
|
[ -z "$preset_file" ] && continue
|
||||||
|
PRESET_COUNT=$((PRESET_COUNT + 1))
|
||||||
|
relname="$(basename "$preset_file")"
|
||||||
|
if grep -qF 'Critique-iterate hint' "$preset_file"; then
|
||||||
|
pass "presets/$relname: 'Critique-iterate hint' block present"
|
||||||
|
else
|
||||||
|
fail "presets/$relname: 'Critique-iterate hint' block missing"
|
||||||
|
fi
|
||||||
|
done < <(find "$PRESET_DIR" -maxdepth 1 -name '*.md' -type f | sort)
|
||||||
|
|
||||||
|
if [ "$PRESET_COUNT" -eq 0 ]; then
|
||||||
|
fail "references/presets/ contains zero .md files — nothing was checked"
|
||||||
|
else
|
||||||
|
pass "presets examined: $PRESET_COUNT"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# (4) 05-critique-iterate.md citation coverage
|
||||||
|
# -------------------------------------------------------
|
||||||
|
echo "--- (4) citation coverage for 05-critique-iterate.md ---"
|
||||||
|
|
||||||
|
if [ ! -f "$CRITIQUE_REF" ]; then
|
||||||
|
fail "references/05-critique-iterate.md missing — cannot check citations"
|
||||||
|
else
|
||||||
|
CITE_COUNT="$(grep -cE "$ANTHROPIC_REGEX" "$CRITIQUE_REF" || true)"
|
||||||
|
if [ -z "$CITE_COUNT" ]; then CITE_COUNT=0; fi
|
||||||
|
if [ "$CITE_COUNT" -ge 1 ]; then
|
||||||
|
pass "05-critique-iterate.md: $CITE_COUNT Anthropic-domain URL citation(s)"
|
||||||
|
else
|
||||||
|
fail "05-critique-iterate.md: zero Anthropic-domain URL citations"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== Summary ==="
|
||||||
|
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||||
|
|
||||||
|
if [ "$FAIL" -gt 0 ]; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
Loading…
Add table
Add a link
Reference in a new issue