feat(claude-design): add SC2 coverage + SC3 citation tests
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
parent
3d143275c1
commit
9882d416b5
2 changed files with 223 additions and 0 deletions
100
plugins/claude-design/tests/test-sc2-artifact-coverage.sh
Executable file
100
plugins/claude-design/tests/test-sc2-artifact-coverage.sh
Executable file
|
|
@ -0,0 +1,100 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# test-sc2-artifact-coverage.sh — Verifies SC2 (per-preset coverage)
|
||||||
|
#
|
||||||
|
# Reads .coverage.md, extracts preset names from the table column 1,
|
||||||
|
# for each preset runs:
|
||||||
|
# grep -rli "<preset>" plugins/claude-design/ --include='*.md' \
|
||||||
|
# --exclude-dir='.claude' --exclude-dir='tests'
|
||||||
|
# and asserts ≥1 file hit.
|
||||||
|
#
|
||||||
|
# The preset list is NOT hardcoded — auto-adapts when .coverage.md changes.
|
||||||
|
#
|
||||||
|
# Usage: bash tests/test-sc2-artifact-coverage.sh
|
||||||
|
# Exit codes: 0 = all presets covered; 1 = at least one preset uncovered
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
LC_ALL=en_US.UTF-8
|
||||||
|
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
YELLOW='\033[1;33m'
|
||||||
|
NC='\033[0m'
|
||||||
|
|
||||||
|
PLUGIN_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
PASS=0
|
||||||
|
FAIL=0
|
||||||
|
WARN=0
|
||||||
|
|
||||||
|
pass() { printf "${GREEN} ✓ %s${NC}\n" "$1"; PASS=$((PASS + 1)); }
|
||||||
|
fail() { printf "${RED} ✗ %s${NC}\n" "$1"; FAIL=$((FAIL + 1)); }
|
||||||
|
warn() { printf "${YELLOW} ⚠ %s${NC}\n" "$1"; WARN=$((WARN + 1)); }
|
||||||
|
|
||||||
|
COVERAGE_FILE="$PLUGIN_ROOT/.coverage.md"
|
||||||
|
|
||||||
|
echo "=== test-sc2-artifact-coverage ==="
|
||||||
|
echo "Plugin root: $PLUGIN_ROOT"
|
||||||
|
echo ".coverage.md: $COVERAGE_FILE"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
if [ ! -f "$COVERAGE_FILE" ]; then
|
||||||
|
fail ".coverage.md missing — cannot verify SC2"
|
||||||
|
echo ""
|
||||||
|
echo "=== Summary ==="
|
||||||
|
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# Extract preset names from .coverage.md table column 1
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# Table rows look like:
|
||||||
|
# | designs | skills/.../presets/designs.md | Evidence grade: ... | https://... |
|
||||||
|
# Skip header (| Preset | ...) and separator (| --- | ...).
|
||||||
|
PRESETS="$(awk -F'|' '
|
||||||
|
/^\| / && NR > 1 {
|
||||||
|
name = $2
|
||||||
|
gsub(/^ +| +$/, "", name)
|
||||||
|
if (name != "Preset" && name !~ /^-+$/ && name != "") {
|
||||||
|
print name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
' "$COVERAGE_FILE")"
|
||||||
|
|
||||||
|
if [ -z "$PRESETS" ]; then
|
||||||
|
fail "no preset names extracted from .coverage.md"
|
||||||
|
echo ""
|
||||||
|
echo "=== Summary ==="
|
||||||
|
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# For each preset, grep for at least one file hit in plugin content
|
||||||
|
# -------------------------------------------------------
|
||||||
|
while IFS= read -r preset; do
|
||||||
|
[ -z "$preset" ] && continue
|
||||||
|
|
||||||
|
HITS="$(grep -rli "$preset" "$PLUGIN_ROOT" \
|
||||||
|
--include='*.md' \
|
||||||
|
--exclude-dir='.claude' \
|
||||||
|
--exclude-dir='tests' \
|
||||||
|
2>/dev/null || true)"
|
||||||
|
|
||||||
|
HIT_COUNT="$(printf '%s\n' "$HITS" | grep -c '.' || true)"
|
||||||
|
if [ -z "$HIT_COUNT" ]; then HIT_COUNT=0; fi
|
||||||
|
|
||||||
|
if [ "$HIT_COUNT" -ge 1 ]; then
|
||||||
|
pass "preset '$preset' covered by $HIT_COUNT file(s)"
|
||||||
|
else
|
||||||
|
fail "preset '$preset' has zero file hits in plugin content"
|
||||||
|
fi
|
||||||
|
done < <(printf '%s\n' "$PRESETS")
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== Summary ==="
|
||||||
|
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||||
|
|
||||||
|
if [ "$FAIL" -gt 0 ]; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
123
plugins/claude-design/tests/test-sc3-citations.sh
Executable file
123
plugins/claude-design/tests/test-sc3-citations.sh
Executable file
|
|
@ -0,0 +1,123 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# test-sc3-citations.sh — Verifies SC3 (Anthropic-domain citation discipline)
|
||||||
|
#
|
||||||
|
# Two checks:
|
||||||
|
# Negative — grep -rnE '\[CITE\]|\[verify\]|\baccording to\b' against
|
||||||
|
# shipped content. Zero hits expected.
|
||||||
|
# Positive — read .coverage.md "Authoritative-claims" bullet list
|
||||||
|
# (awk on '^- ' prefix), then for each file ensure ≥1
|
||||||
|
# Anthropic-domain URL citation is present.
|
||||||
|
#
|
||||||
|
# Excludes .claude/projects/** and tests/ from greps.
|
||||||
|
#
|
||||||
|
# Anthropic-domain URL regex (positive):
|
||||||
|
# https?://(docs\.anthropic\.com|anthropic\.com|github\.com/anthropics
|
||||||
|
# |claude\.com|support\.claude\.com|platform\.claude\.com)
|
||||||
|
#
|
||||||
|
# Usage: bash tests/test-sc3-citations.sh
|
||||||
|
# Exit codes: 0 = pass; 1 = at least one FAIL
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
LC_ALL=en_US.UTF-8
|
||||||
|
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
YELLOW='\033[1;33m'
|
||||||
|
NC='\033[0m'
|
||||||
|
|
||||||
|
PLUGIN_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
PASS=0
|
||||||
|
FAIL=0
|
||||||
|
WARN=0
|
||||||
|
|
||||||
|
pass() { printf "${GREEN} ✓ %s${NC}\n" "$1"; PASS=$((PASS + 1)); }
|
||||||
|
fail() { printf "${RED} ✗ %s${NC}\n" "$1"; FAIL=$((FAIL + 1)); }
|
||||||
|
warn() { printf "${YELLOW} ⚠ %s${NC}\n" "$1"; WARN=$((WARN + 1)); }
|
||||||
|
|
||||||
|
COVERAGE_FILE="$PLUGIN_ROOT/.coverage.md"
|
||||||
|
|
||||||
|
echo "=== test-sc3-citations ==="
|
||||||
|
echo "Plugin root: $PLUGIN_ROOT"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# Negative grep: \[CITE\], \[verify\], \baccording to\b
|
||||||
|
# -------------------------------------------------------
|
||||||
|
echo "--- negative grep: forbidden placeholders ---"
|
||||||
|
|
||||||
|
NEG_HITS="$(grep -rnE '\[CITE\]|\[verify\]|\baccording to\b' \
|
||||||
|
"$PLUGIN_ROOT" \
|
||||||
|
--include='*.md' \
|
||||||
|
--exclude-dir='.claude' \
|
||||||
|
--exclude-dir='tests' \
|
||||||
|
2>/dev/null || true)"
|
||||||
|
|
||||||
|
if [ -z "$NEG_HITS" ]; then
|
||||||
|
pass "no forbidden placeholders ([CITE], [verify], 'according to') in shipped content"
|
||||||
|
else
|
||||||
|
while IFS= read -r hit; do
|
||||||
|
fail "forbidden placeholder in shipped content: $hit"
|
||||||
|
done < <(printf '%s\n' "$NEG_HITS")
|
||||||
|
fi
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# Positive grep: Authoritative-claims files have Anthropic-domain URLs
|
||||||
|
# -------------------------------------------------------
|
||||||
|
echo "--- positive grep: Authoritative-claims citation coverage ---"
|
||||||
|
|
||||||
|
if [ ! -f "$COVERAGE_FILE" ]; then
|
||||||
|
fail ".coverage.md missing — cannot read Authoritative-claims registry"
|
||||||
|
echo ""
|
||||||
|
echo "=== Summary ==="
|
||||||
|
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Extract bullet-list paths under the "Authoritative-claims" section
|
||||||
|
AUTH_FILES="$(awk '
|
||||||
|
/^## Authoritative-claims files/ { capture = 1; next }
|
||||||
|
capture && /^## / { exit }
|
||||||
|
capture && /^- / {
|
||||||
|
sub(/^- /, "", $0)
|
||||||
|
print $0
|
||||||
|
}
|
||||||
|
' "$COVERAGE_FILE")"
|
||||||
|
|
||||||
|
if [ -z "$AUTH_FILES" ]; then
|
||||||
|
fail "no Authoritative-claims files extracted from .coverage.md"
|
||||||
|
echo ""
|
||||||
|
echo "=== Summary ==="
|
||||||
|
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
ANTHROPIC_REGEX='https?://(docs\.anthropic\.com|anthropic\.com|github\.com/anthropics|claude\.com|support\.claude\.com|platform\.claude\.com)'
|
||||||
|
|
||||||
|
while IFS= read -r relpath; do
|
||||||
|
[ -z "$relpath" ] && continue
|
||||||
|
fpath="$PLUGIN_ROOT/$relpath"
|
||||||
|
|
||||||
|
if [ ! -f "$fpath" ]; then
|
||||||
|
fail "Authoritative-claims file missing: $relpath"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
HIT_COUNT="$(grep -cE "$ANTHROPIC_REGEX" "$fpath" || true)"
|
||||||
|
if [ -z "$HIT_COUNT" ]; then HIT_COUNT=0; fi
|
||||||
|
|
||||||
|
if [ "$HIT_COUNT" -ge 1 ]; then
|
||||||
|
pass "$relpath: $HIT_COUNT Anthropic-domain URL citation(s)"
|
||||||
|
else
|
||||||
|
fail "$relpath: zero Anthropic-domain URL citations (anthropic.com expected)"
|
||||||
|
fi
|
||||||
|
done < <(printf '%s\n' "$AUTH_FILES")
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== Summary ==="
|
||||||
|
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||||
|
|
||||||
|
if [ "$FAIL" -gt 0 ]; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
Loading…
Add table
Add a link
Reference in a new issue