diff --git a/plugins/claude-design/tests/test-sc1-dogfood-log.sh b/plugins/claude-design/tests/test-sc1-dogfood-log.sh new file mode 100755 index 0000000..885db80 --- /dev/null +++ b/plugins/claude-design/tests/test-sc1-dogfood-log.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +# test-sc1-dogfood-log.sh — Verifies SC1 (operator-attested dogfood log) in REMEMBER.md +# +# Usage: +# bash tests/test-sc1-dogfood-log.sh # missing block = WARN, exit 0 +# bash tests/test-sc1-dogfood-log.sh --strict # missing block = FAIL, exit 1 +# +# Expects in REMEMBER.md (plugin root, gitignored): +# - A fenced section with heading `## Dogfood log — v0.1 slides run` +# - Five mechanically-checkable fields inside the section: +# artifact_type: +# refine_rounds: +# final_prompt: +# ``` +# +# ``` +# shipped: yes (or `shipped: equivalent`) +# comparison_to_unaided: +# +# REMEMBER.md is gitignored — this evidence is local-only. The script +# validates format only; the outcome judgement is operator-attested. + +set -euo pipefail +LC_ALL=en_US.UTF-8 + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +PLUGIN_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PASS=0 +FAIL=0 +WARN=0 + +pass() { printf "${GREEN} ✓ %s${NC}\n" "$1"; PASS=$((PASS + 1)); } +fail() { printf "${RED} ✗ %s${NC}\n" "$1"; FAIL=$((FAIL + 1)); } +warn() { printf "${YELLOW} ⚠ %s${NC}\n" "$1"; WARN=$((WARN + 1)); } + +STRICT=false +for arg in "$@"; do + case "$arg" in + --strict) STRICT=true ;; + esac +done + +echo "=== test-sc1-dogfood-log ===" +echo "Plugin root: $PLUGIN_ROOT" +echo "Strict mode: $STRICT" +echo "" + +REMEMBER_FILE="$PLUGIN_ROOT/REMEMBER.md" +COVERAGE_FILE="$PLUGIN_ROOT/.coverage.md" + +# ------------------------------------------------------- +# Locate REMEMBER.md +# ------------------------------------------------------- +if [ ! -f "$REMEMBER_FILE" ]; then + if $STRICT; then + fail "REMEMBER.md missing (strict mode — required)" + echo "" + echo "=== Summary ===" + printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN" + exit 1 + else + warn "REMEMBER.md missing (advisory until operator dogfood step)" + echo "" + echo "=== Summary ===" + printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN" + exit 0 + fi +fi + +# ------------------------------------------------------- +# Extract fenced block between dogfood heading and next H2 (or EOF) +# ------------------------------------------------------- +BLOCK="$(awk ' + /^## Dogfood log — v0\.1 slides run$/ { capture = 1; next } + capture && /^## / { exit } + capture { print } +' "$REMEMBER_FILE")" + +if [ -z "$BLOCK" ]; then + if $STRICT; then + fail "REMEMBER.md missing dogfood block '## Dogfood log — v0.1 slides run' (strict mode — required)" + echo "" + echo "=== Summary ===" + printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN" + exit 1 + else + warn "REMEMBER.md missing dogfood block (advisory until operator dogfood step)" + echo "" + echo "=== Summary ===" + printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN" + exit 0 + fi +fi + +pass "found '## Dogfood log — v0.1 slides run' block" + +# ------------------------------------------------------- +# Field 1: artifact_type — must match a preset name from .coverage.md +# ------------------------------------------------------- +ARTIFACT_TYPE="$(printf '%s\n' "$BLOCK" | awk -F': *' '/^artifact_type:/ { print $2; exit }' | tr -d '[:space:]')" + +if [ -z "$ARTIFACT_TYPE" ]; then + fail "artifact_type: field missing or empty" +else + # extract preset names from .coverage.md table column 1 + if [ -f "$COVERAGE_FILE" ]; then + PRESETS="$(awk -F'|' ' + /^\| [a-z]/ { gsub(/^ +| +$/, "", $2); print $2 } + ' "$COVERAGE_FILE")" + + FOUND=false + while IFS= read -r preset; do + [ -z "$preset" ] && continue + if [ "$preset" = "$ARTIFACT_TYPE" ]; then + FOUND=true + break + fi + done < <(printf '%s\n' "$PRESETS") + + if $FOUND; then + pass "artifact_type='$ARTIFACT_TYPE' matches a preset in .coverage.md" + else + fail "artifact_type='$ARTIFACT_TYPE' does not match any preset in .coverage.md" + fi + else + fail ".coverage.md missing — cannot validate artifact_type" + fi +fi + +# ------------------------------------------------------- +# Field 2: refine_rounds — integer +# ------------------------------------------------------- +REFINE_ROUNDS_LINE="$(printf '%s\n' "$BLOCK" | grep -E '^refine_rounds:[[:space:]]*[0-9]+[[:space:]]*$' || true)" +if [ -n "$REFINE_ROUNDS_LINE" ]; then + pass "refine_rounds: matches integer regex" +else + fail "refine_rounds: missing or not an integer" +fi + +# ------------------------------------------------------- +# Field 3: final_prompt: followed by non-empty fenced code block +# ------------------------------------------------------- +HAS_FINAL_PROMPT="$(printf '%s\n' "$BLOCK" | grep -c '^final_prompt:' || true)" +if [ "$HAS_FINAL_PROMPT" -ge 1 ]; then + # check that a fenced code block (```) appears after final_prompt: + FENCE_AFTER="$(awk ' + /^final_prompt:/ { found = 1; next } + found && /^```/ { fence_open = !fence_open; if (fence_open) { in_fence = 1 } else { exit } } + found && in_fence && fence_open && /./ { content_lines++ } + END { print content_lines + 0 } + ' <<<"$BLOCK")" + if [ -z "$FENCE_AFTER" ]; then FENCE_AFTER=0; fi + if [ "$FENCE_AFTER" -ge 1 ]; then + pass "final_prompt: followed by non-empty fenced code block ($FENCE_AFTER content line(s))" + else + fail "final_prompt: not followed by a non-empty fenced code block" + fi +else + fail "final_prompt: field missing" +fi + +# ------------------------------------------------------- +# Field 4: shipped — yes or equivalent +# ------------------------------------------------------- +SHIPPED_LINE="$(printf '%s\n' "$BLOCK" | grep -E '^shipped:[[:space:]]*(yes|equivalent)[[:space:]]*$' || true)" +if [ -n "$SHIPPED_LINE" ]; then + pass "shipped: matches 'yes' or 'equivalent'" +else + fail "shipped: missing or not 'yes'/'equivalent'" +fi + +# ------------------------------------------------------- +# Field 5: comparison_to_unaided — non-empty sentence >=10 chars ending with . +# ------------------------------------------------------- +COMP_LINE="$(printf '%s\n' "$BLOCK" | awk -F': *' '/^comparison_to_unaided:/ { for (i=2;i<=NF;i++) printf "%s%s", $i, (i=10)" +elif [ "${COMP_TRIMMED: -1}" != "." ]; then + fail "comparison_to_unaided: does not end with '.'" +else + pass "comparison_to_unaided: non-empty, $COMP_LEN chars, ends with '.'" +fi + +# ------------------------------------------------------- +# Summary +# ------------------------------------------------------- +echo "" +echo "=== Summary ===" +printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN" + +if [ "$FAIL" -gt 0 ]; then + exit 1 +fi +exit 0 diff --git a/plugins/claude-design/tests/test-skill-triggers.sh b/plugins/claude-design/tests/test-skill-triggers.sh new file mode 100755 index 0000000..a82b7f0 --- /dev/null +++ b/plugins/claude-design/tests/test-skill-triggers.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# test-skill-triggers.sh — Verifies skill description quality +# +# Honest limit: this only verifies strings are present in SKILL.md description. +# It cannot prove Claude Code's orchestrator fires the skill on those prompts. +# Runtime auto-fire validation is the operator's dogfood step (SC1). +# +# Checks: +# - SKILL.md frontmatter has 'description:' field +# - description block (from `description: |` to the closing `---`) is >=400 chars +# - if .triggers.txt exists, every phrase in it appears in SKILL.md description +# +# Usage: bash tests/test-skill-triggers.sh +# Exit codes: 0 = pass; 1 = at least one FAIL + +set -euo pipefail +LC_ALL=en_US.UTF-8 + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +PLUGIN_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PASS=0 +FAIL=0 +WARN=0 + +pass() { printf "${GREEN} ✓ %s${NC}\n" "$1"; PASS=$((PASS + 1)); } +fail() { printf "${RED} ✗ %s${NC}\n" "$1"; FAIL=$((FAIL + 1)); } +warn() { printf "${YELLOW} ⚠ %s${NC}\n" "$1"; WARN=$((WARN + 1)); } + +echo "=== test-skill-triggers ===" +echo "Plugin root: $PLUGIN_ROOT" +echo "" + +# ------------------------------------------------------- +# Iterate over every SKILL.md +# ------------------------------------------------------- +SKILL_COUNT=0 +for skill_file in "$PLUGIN_ROOT"/skills/*/SKILL.md; do + [ -f "$skill_file" ] || continue + SKILL_COUNT=$((SKILL_COUNT + 1)) + + skill_dir="$(dirname "$skill_file")" + skill_name="$(basename "$skill_dir")" + + echo "--- $skill_name ---" + + # ------------------------ + # Frontmatter check + # ------------------------ + first_line="$(head -n 1 "$skill_file")" + if [ "$first_line" != "---" ]; then + fail "$skill_name/SKILL.md: missing frontmatter delimiter on line 1" + echo "" + continue + fi + + # ------------------------ + # Description >=400 chars (Triggers on: enumeration counts) + # ------------------------ + desc_block_chars="$(awk '/^description: \|/,/^---$/' "$skill_file" | wc -c | tr -d '[:space:]')" + if [ -z "$desc_block_chars" ]; then desc_block_chars=0; fi + + if [ "$desc_block_chars" -ge 400 ]; then + pass "$skill_name: description block is $desc_block_chars chars (>=400)" + else + fail "$skill_name: description block is $desc_block_chars chars (<400 required)" + fi + + # ------------------------ + # Trigger-phrase coverage (.triggers.txt) + # ------------------------ + triggers_file="$skill_dir/.triggers.txt" + + if [ ! -f "$triggers_file" ]; then + warn "$skill_name: .triggers.txt missing (advisory — operator may want one)" + echo "" + continue + fi + + trigger_count="$(grep -cE '.' "$triggers_file" || true)" + if [ -z "$trigger_count" ] || [ "$trigger_count" -lt 8 ]; then + fail "$skill_name/.triggers.txt: only $trigger_count phrase(s) (>=8 required)" + else + pass "$skill_name/.triggers.txt: $trigger_count phrase(s)" + fi + + # check each phrase appears in SKILL.md description block + MISSING=0 + while IFS= read -r phrase; do + [ -z "$phrase" ] && continue + if ! grep -qF "$phrase" "$skill_file"; then + fail "$skill_name: trigger phrase missing from SKILL.md description: '$phrase'" + MISSING=$((MISSING + 1)) + fi + done < "$triggers_file" + + if [ "$MISSING" -eq 0 ]; then + pass "$skill_name: all $trigger_count trigger phrase(s) appear in SKILL.md" + fi + + echo "" +done + +if [ "$SKILL_COUNT" -eq 0 ]; then + fail "no SKILL.md found under skills/*/" +fi + +# ------------------------------------------------------- +# Summary +# ------------------------------------------------------- +echo "=== Summary ===" +printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN" + +if [ "$FAIL" -gt 0 ]; then + exit 1 +fi +exit 0 + +# Triggers on: documented in each skill's .triggers.txt sibling file.