feat(claude-design): add SC1 dogfood-log + skill-triggers tests
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
parent
3dc0414948
commit
3d143275c1
2 changed files with 325 additions and 0 deletions
203
plugins/claude-design/tests/test-sc1-dogfood-log.sh
Executable file
203
plugins/claude-design/tests/test-sc1-dogfood-log.sh
Executable file
|
|
@ -0,0 +1,203 @@
|
|||
#!/usr/bin/env bash
|
||||
# test-sc1-dogfood-log.sh — Verifies SC1 (operator-attested dogfood log) in REMEMBER.md
|
||||
#
|
||||
# Usage:
|
||||
# bash tests/test-sc1-dogfood-log.sh # missing block = WARN, exit 0
|
||||
# bash tests/test-sc1-dogfood-log.sh --strict # missing block = FAIL, exit 1
|
||||
#
|
||||
# Expects in REMEMBER.md (plugin root, gitignored):
|
||||
# - A fenced section with heading `## Dogfood log — v0.1 slides run`
|
||||
# - Five mechanically-checkable fields inside the section:
|
||||
# artifact_type: <preset-name from .coverage.md>
|
||||
# refine_rounds: <integer>
|
||||
# final_prompt:
|
||||
# ```
|
||||
# <non-empty prompt content>
|
||||
# ```
|
||||
# shipped: yes (or `shipped: equivalent`)
|
||||
# comparison_to_unaided: <one sentence ending with .>
|
||||
#
|
||||
# REMEMBER.md is gitignored — this evidence is local-only. The script
|
||||
# validates format only; the outcome judgement is operator-attested.
|
||||
|
||||
set -euo pipefail
|
||||
LC_ALL=en_US.UTF-8
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
PLUGIN_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
PASS=0
|
||||
FAIL=0
|
||||
WARN=0
|
||||
|
||||
pass() { printf "${GREEN} ✓ %s${NC}\n" "$1"; PASS=$((PASS + 1)); }
|
||||
fail() { printf "${RED} ✗ %s${NC}\n" "$1"; FAIL=$((FAIL + 1)); }
|
||||
warn() { printf "${YELLOW} ⚠ %s${NC}\n" "$1"; WARN=$((WARN + 1)); }
|
||||
|
||||
STRICT=false
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--strict) STRICT=true ;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "=== test-sc1-dogfood-log ==="
|
||||
echo "Plugin root: $PLUGIN_ROOT"
|
||||
echo "Strict mode: $STRICT"
|
||||
echo ""
|
||||
|
||||
REMEMBER_FILE="$PLUGIN_ROOT/REMEMBER.md"
|
||||
COVERAGE_FILE="$PLUGIN_ROOT/.coverage.md"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Locate REMEMBER.md
|
||||
# -------------------------------------------------------
|
||||
if [ ! -f "$REMEMBER_FILE" ]; then
|
||||
if $STRICT; then
|
||||
fail "REMEMBER.md missing (strict mode — required)"
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||
exit 1
|
||||
else
|
||||
warn "REMEMBER.md missing (advisory until operator dogfood step)"
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Extract fenced block between dogfood heading and next H2 (or EOF)
|
||||
# -------------------------------------------------------
|
||||
BLOCK="$(awk '
|
||||
/^## Dogfood log — v0\.1 slides run$/ { capture = 1; next }
|
||||
capture && /^## / { exit }
|
||||
capture { print }
|
||||
' "$REMEMBER_FILE")"
|
||||
|
||||
if [ -z "$BLOCK" ]; then
|
||||
if $STRICT; then
|
||||
fail "REMEMBER.md missing dogfood block '## Dogfood log — v0.1 slides run' (strict mode — required)"
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||
exit 1
|
||||
else
|
||||
warn "REMEMBER.md missing dogfood block (advisory until operator dogfood step)"
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
pass "found '## Dogfood log — v0.1 slides run' block"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Field 1: artifact_type — must match a preset name from .coverage.md
|
||||
# -------------------------------------------------------
|
||||
ARTIFACT_TYPE="$(printf '%s\n' "$BLOCK" | awk -F': *' '/^artifact_type:/ { print $2; exit }' | tr -d '[:space:]')"
|
||||
|
||||
if [ -z "$ARTIFACT_TYPE" ]; then
|
||||
fail "artifact_type: field missing or empty"
|
||||
else
|
||||
# extract preset names from .coverage.md table column 1
|
||||
if [ -f "$COVERAGE_FILE" ]; then
|
||||
PRESETS="$(awk -F'|' '
|
||||
/^\| [a-z]/ { gsub(/^ +| +$/, "", $2); print $2 }
|
||||
' "$COVERAGE_FILE")"
|
||||
|
||||
FOUND=false
|
||||
while IFS= read -r preset; do
|
||||
[ -z "$preset" ] && continue
|
||||
if [ "$preset" = "$ARTIFACT_TYPE" ]; then
|
||||
FOUND=true
|
||||
break
|
||||
fi
|
||||
done < <(printf '%s\n' "$PRESETS")
|
||||
|
||||
if $FOUND; then
|
||||
pass "artifact_type='$ARTIFACT_TYPE' matches a preset in .coverage.md"
|
||||
else
|
||||
fail "artifact_type='$ARTIFACT_TYPE' does not match any preset in .coverage.md"
|
||||
fi
|
||||
else
|
||||
fail ".coverage.md missing — cannot validate artifact_type"
|
||||
fi
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Field 2: refine_rounds — integer
|
||||
# -------------------------------------------------------
|
||||
REFINE_ROUNDS_LINE="$(printf '%s\n' "$BLOCK" | grep -E '^refine_rounds:[[:space:]]*[0-9]+[[:space:]]*$' || true)"
|
||||
if [ -n "$REFINE_ROUNDS_LINE" ]; then
|
||||
pass "refine_rounds: matches integer regex"
|
||||
else
|
||||
fail "refine_rounds: missing or not an integer"
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Field 3: final_prompt: followed by non-empty fenced code block
|
||||
# -------------------------------------------------------
|
||||
HAS_FINAL_PROMPT="$(printf '%s\n' "$BLOCK" | grep -c '^final_prompt:' || true)"
|
||||
if [ "$HAS_FINAL_PROMPT" -ge 1 ]; then
|
||||
# check that a fenced code block (```) appears after final_prompt:
|
||||
FENCE_AFTER="$(awk '
|
||||
/^final_prompt:/ { found = 1; next }
|
||||
found && /^```/ { fence_open = !fence_open; if (fence_open) { in_fence = 1 } else { exit } }
|
||||
found && in_fence && fence_open && /./ { content_lines++ }
|
||||
END { print content_lines + 0 }
|
||||
' <<<"$BLOCK")"
|
||||
if [ -z "$FENCE_AFTER" ]; then FENCE_AFTER=0; fi
|
||||
if [ "$FENCE_AFTER" -ge 1 ]; then
|
||||
pass "final_prompt: followed by non-empty fenced code block ($FENCE_AFTER content line(s))"
|
||||
else
|
||||
fail "final_prompt: not followed by a non-empty fenced code block"
|
||||
fi
|
||||
else
|
||||
fail "final_prompt: field missing"
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Field 4: shipped — yes or equivalent
|
||||
# -------------------------------------------------------
|
||||
SHIPPED_LINE="$(printf '%s\n' "$BLOCK" | grep -E '^shipped:[[:space:]]*(yes|equivalent)[[:space:]]*$' || true)"
|
||||
if [ -n "$SHIPPED_LINE" ]; then
|
||||
pass "shipped: matches 'yes' or 'equivalent'"
|
||||
else
|
||||
fail "shipped: missing or not 'yes'/'equivalent'"
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Field 5: comparison_to_unaided — non-empty sentence >=10 chars ending with .
|
||||
# -------------------------------------------------------
|
||||
COMP_LINE="$(printf '%s\n' "$BLOCK" | awk -F': *' '/^comparison_to_unaided:/ { for (i=2;i<=NF;i++) printf "%s%s", $i, (i<NF?": ":""); print ""; exit }')"
|
||||
COMP_TRIMMED="$(printf '%s' "$COMP_LINE" | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//')"
|
||||
COMP_LEN="${#COMP_TRIMMED}"
|
||||
|
||||
if [ -z "$COMP_TRIMMED" ]; then
|
||||
fail "comparison_to_unaided: field missing or empty"
|
||||
elif [ "$COMP_LEN" -lt 10 ]; then
|
||||
fail "comparison_to_unaided: too short ($COMP_LEN chars; need >=10)"
|
||||
elif [ "${COMP_TRIMMED: -1}" != "." ]; then
|
||||
fail "comparison_to_unaided: does not end with '.'"
|
||||
else
|
||||
pass "comparison_to_unaided: non-empty, $COMP_LEN chars, ends with '.'"
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Summary
|
||||
# -------------------------------------------------------
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
122
plugins/claude-design/tests/test-skill-triggers.sh
Executable file
122
plugins/claude-design/tests/test-skill-triggers.sh
Executable file
|
|
@ -0,0 +1,122 @@
|
|||
#!/usr/bin/env bash
|
||||
# test-skill-triggers.sh — Verifies skill description quality
|
||||
#
|
||||
# Honest limit: this only verifies strings are present in SKILL.md description.
|
||||
# It cannot prove Claude Code's orchestrator fires the skill on those prompts.
|
||||
# Runtime auto-fire validation is the operator's dogfood step (SC1).
|
||||
#
|
||||
# Checks:
|
||||
# - SKILL.md frontmatter has 'description:' field
|
||||
# - description block (from `description: |` to the closing `---`) is >=400 chars
|
||||
# - if .triggers.txt exists, every phrase in it appears in SKILL.md description
|
||||
#
|
||||
# Usage: bash tests/test-skill-triggers.sh
|
||||
# Exit codes: 0 = pass; 1 = at least one FAIL
|
||||
|
||||
set -euo pipefail
|
||||
LC_ALL=en_US.UTF-8
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
PLUGIN_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
PASS=0
|
||||
FAIL=0
|
||||
WARN=0
|
||||
|
||||
pass() { printf "${GREEN} ✓ %s${NC}\n" "$1"; PASS=$((PASS + 1)); }
|
||||
fail() { printf "${RED} ✗ %s${NC}\n" "$1"; FAIL=$((FAIL + 1)); }
|
||||
warn() { printf "${YELLOW} ⚠ %s${NC}\n" "$1"; WARN=$((WARN + 1)); }
|
||||
|
||||
echo "=== test-skill-triggers ==="
|
||||
echo "Plugin root: $PLUGIN_ROOT"
|
||||
echo ""
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Iterate over every SKILL.md
|
||||
# -------------------------------------------------------
|
||||
SKILL_COUNT=0
|
||||
for skill_file in "$PLUGIN_ROOT"/skills/*/SKILL.md; do
|
||||
[ -f "$skill_file" ] || continue
|
||||
SKILL_COUNT=$((SKILL_COUNT + 1))
|
||||
|
||||
skill_dir="$(dirname "$skill_file")"
|
||||
skill_name="$(basename "$skill_dir")"
|
||||
|
||||
echo "--- $skill_name ---"
|
||||
|
||||
# ------------------------
|
||||
# Frontmatter check
|
||||
# ------------------------
|
||||
first_line="$(head -n 1 "$skill_file")"
|
||||
if [ "$first_line" != "---" ]; then
|
||||
fail "$skill_name/SKILL.md: missing frontmatter delimiter on line 1"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
# ------------------------
|
||||
# Description >=400 chars (Triggers on: enumeration counts)
|
||||
# ------------------------
|
||||
desc_block_chars="$(awk '/^description: \|/,/^---$/' "$skill_file" | wc -c | tr -d '[:space:]')"
|
||||
if [ -z "$desc_block_chars" ]; then desc_block_chars=0; fi
|
||||
|
||||
if [ "$desc_block_chars" -ge 400 ]; then
|
||||
pass "$skill_name: description block is $desc_block_chars chars (>=400)"
|
||||
else
|
||||
fail "$skill_name: description block is $desc_block_chars chars (<400 required)"
|
||||
fi
|
||||
|
||||
# ------------------------
|
||||
# Trigger-phrase coverage (.triggers.txt)
|
||||
# ------------------------
|
||||
triggers_file="$skill_dir/.triggers.txt"
|
||||
|
||||
if [ ! -f "$triggers_file" ]; then
|
||||
warn "$skill_name: .triggers.txt missing (advisory — operator may want one)"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
trigger_count="$(grep -cE '.' "$triggers_file" || true)"
|
||||
if [ -z "$trigger_count" ] || [ "$trigger_count" -lt 8 ]; then
|
||||
fail "$skill_name/.triggers.txt: only $trigger_count phrase(s) (>=8 required)"
|
||||
else
|
||||
pass "$skill_name/.triggers.txt: $trigger_count phrase(s)"
|
||||
fi
|
||||
|
||||
# check each phrase appears in SKILL.md description block
|
||||
MISSING=0
|
||||
while IFS= read -r phrase; do
|
||||
[ -z "$phrase" ] && continue
|
||||
if ! grep -qF "$phrase" "$skill_file"; then
|
||||
fail "$skill_name: trigger phrase missing from SKILL.md description: '$phrase'"
|
||||
MISSING=$((MISSING + 1))
|
||||
fi
|
||||
done < "$triggers_file"
|
||||
|
||||
if [ "$MISSING" -eq 0 ]; then
|
||||
pass "$skill_name: all $trigger_count trigger phrase(s) appear in SKILL.md"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
if [ "$SKILL_COUNT" -eq 0 ]; then
|
||||
fail "no SKILL.md found under skills/*/"
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Summary
|
||||
# -------------------------------------------------------
|
||||
echo "=== Summary ==="
|
||||
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
|
||||
# Triggers on: documented in each skill's .triggers.txt sibling file.
|
||||
Loading…
Add table
Add a link
Reference in a new issue