feat(claude-design): add SC2 coverage + SC3 citation tests
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
parent
3d143275c1
commit
9882d416b5
2 changed files with 223 additions and 0 deletions
100
plugins/claude-design/tests/test-sc2-artifact-coverage.sh
Executable file
100
plugins/claude-design/tests/test-sc2-artifact-coverage.sh
Executable file
|
|
@ -0,0 +1,100 @@
|
|||
#!/usr/bin/env bash
|
||||
# test-sc2-artifact-coverage.sh — Verifies SC2 (per-preset coverage)
|
||||
#
|
||||
# Reads .coverage.md, extracts preset names from the table column 1,
|
||||
# for each preset runs:
|
||||
# grep -rli "<preset>" plugins/claude-design/ --include='*.md' \
|
||||
# --exclude-dir='.claude' --exclude-dir='tests'
|
||||
# and asserts ≥1 file hit.
|
||||
#
|
||||
# The preset list is NOT hardcoded — auto-adapts when .coverage.md changes.
|
||||
#
|
||||
# Usage: bash tests/test-sc2-artifact-coverage.sh
|
||||
# Exit codes: 0 = all presets covered; 1 = at least one preset uncovered
|
||||
|
||||
set -euo pipefail
|
||||
LC_ALL=en_US.UTF-8
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
PLUGIN_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
PASS=0
|
||||
FAIL=0
|
||||
WARN=0
|
||||
|
||||
pass() { printf "${GREEN} ✓ %s${NC}\n" "$1"; PASS=$((PASS + 1)); }
|
||||
fail() { printf "${RED} ✗ %s${NC}\n" "$1"; FAIL=$((FAIL + 1)); }
|
||||
warn() { printf "${YELLOW} ⚠ %s${NC}\n" "$1"; WARN=$((WARN + 1)); }
|
||||
|
||||
COVERAGE_FILE="$PLUGIN_ROOT/.coverage.md"
|
||||
|
||||
echo "=== test-sc2-artifact-coverage ==="
|
||||
echo "Plugin root: $PLUGIN_ROOT"
|
||||
echo ".coverage.md: $COVERAGE_FILE"
|
||||
echo ""
|
||||
|
||||
if [ ! -f "$COVERAGE_FILE" ]; then
|
||||
fail ".coverage.md missing — cannot verify SC2"
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Extract preset names from .coverage.md table column 1
|
||||
# -------------------------------------------------------
|
||||
# Table rows look like:
|
||||
# | designs | skills/.../presets/designs.md | Evidence grade: ... | https://... |
|
||||
# Skip header (| Preset | ...) and separator (| --- | ...).
|
||||
PRESETS="$(awk -F'|' '
|
||||
/^\| / && NR > 1 {
|
||||
name = $2
|
||||
gsub(/^ +| +$/, "", name)
|
||||
if (name != "Preset" && name !~ /^-+$/ && name != "") {
|
||||
print name
|
||||
}
|
||||
}
|
||||
' "$COVERAGE_FILE")"
|
||||
|
||||
if [ -z "$PRESETS" ]; then
|
||||
fail "no preset names extracted from .coverage.md"
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------
|
||||
# For each preset, grep for at least one file hit in plugin content
|
||||
# -------------------------------------------------------
|
||||
while IFS= read -r preset; do
|
||||
[ -z "$preset" ] && continue
|
||||
|
||||
HITS="$(grep -rli "$preset" "$PLUGIN_ROOT" \
|
||||
--include='*.md' \
|
||||
--exclude-dir='.claude' \
|
||||
--exclude-dir='tests' \
|
||||
2>/dev/null || true)"
|
||||
|
||||
HIT_COUNT="$(printf '%s\n' "$HITS" | grep -c '.' || true)"
|
||||
if [ -z "$HIT_COUNT" ]; then HIT_COUNT=0; fi
|
||||
|
||||
if [ "$HIT_COUNT" -ge 1 ]; then
|
||||
pass "preset '$preset' covered by $HIT_COUNT file(s)"
|
||||
else
|
||||
fail "preset '$preset' has zero file hits in plugin content"
|
||||
fi
|
||||
done < <(printf '%s\n' "$PRESETS")
|
||||
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
123
plugins/claude-design/tests/test-sc3-citations.sh
Executable file
123
plugins/claude-design/tests/test-sc3-citations.sh
Executable file
|
|
@ -0,0 +1,123 @@
|
|||
#!/usr/bin/env bash
|
||||
# test-sc3-citations.sh — Verifies SC3 (Anthropic-domain citation discipline)
|
||||
#
|
||||
# Two checks:
|
||||
# Negative — grep -rnE '\[CITE\]|\[verify\]|\baccording to\b' against
|
||||
# shipped content. Zero hits expected.
|
||||
# Positive — read .coverage.md "Authoritative-claims" bullet list
|
||||
# (awk on '^- ' prefix), then for each file ensure ≥1
|
||||
# Anthropic-domain URL citation is present.
|
||||
#
|
||||
# Excludes .claude/projects/** and tests/ from greps.
|
||||
#
|
||||
# Anthropic-domain URL regex (positive):
|
||||
# https?://(docs\.anthropic\.com|anthropic\.com|github\.com/anthropics
|
||||
# |claude\.com|support\.claude\.com|platform\.claude\.com)
|
||||
#
|
||||
# Usage: bash tests/test-sc3-citations.sh
|
||||
# Exit codes: 0 = pass; 1 = at least one FAIL
|
||||
|
||||
set -euo pipefail
|
||||
LC_ALL=en_US.UTF-8
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
PLUGIN_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
PASS=0
|
||||
FAIL=0
|
||||
WARN=0
|
||||
|
||||
pass() { printf "${GREEN} ✓ %s${NC}\n" "$1"; PASS=$((PASS + 1)); }
|
||||
fail() { printf "${RED} ✗ %s${NC}\n" "$1"; FAIL=$((FAIL + 1)); }
|
||||
warn() { printf "${YELLOW} ⚠ %s${NC}\n" "$1"; WARN=$((WARN + 1)); }
|
||||
|
||||
COVERAGE_FILE="$PLUGIN_ROOT/.coverage.md"
|
||||
|
||||
echo "=== test-sc3-citations ==="
|
||||
echo "Plugin root: $PLUGIN_ROOT"
|
||||
echo ""
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Negative grep: \[CITE\], \[verify\], \baccording to\b
|
||||
# -------------------------------------------------------
|
||||
echo "--- negative grep: forbidden placeholders ---"
|
||||
|
||||
NEG_HITS="$(grep -rnE '\[CITE\]|\[verify\]|\baccording to\b' \
|
||||
"$PLUGIN_ROOT" \
|
||||
--include='*.md' \
|
||||
--exclude-dir='.claude' \
|
||||
--exclude-dir='tests' \
|
||||
2>/dev/null || true)"
|
||||
|
||||
if [ -z "$NEG_HITS" ]; then
|
||||
pass "no forbidden placeholders ([CITE], [verify], 'according to') in shipped content"
|
||||
else
|
||||
while IFS= read -r hit; do
|
||||
fail "forbidden placeholder in shipped content: $hit"
|
||||
done < <(printf '%s\n' "$NEG_HITS")
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Positive grep: Authoritative-claims files have Anthropic-domain URLs
|
||||
# -------------------------------------------------------
|
||||
echo "--- positive grep: Authoritative-claims citation coverage ---"
|
||||
|
||||
if [ ! -f "$COVERAGE_FILE" ]; then
|
||||
fail ".coverage.md missing — cannot read Authoritative-claims registry"
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Extract bullet-list paths under the "Authoritative-claims" section
|
||||
AUTH_FILES="$(awk '
|
||||
/^## Authoritative-claims files/ { capture = 1; next }
|
||||
capture && /^## / { exit }
|
||||
capture && /^- / {
|
||||
sub(/^- /, "", $0)
|
||||
print $0
|
||||
}
|
||||
' "$COVERAGE_FILE")"
|
||||
|
||||
if [ -z "$AUTH_FILES" ]; then
|
||||
fail "no Authoritative-claims files extracted from .coverage.md"
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ANTHROPIC_REGEX='https?://(docs\.anthropic\.com|anthropic\.com|github\.com/anthropics|claude\.com|support\.claude\.com|platform\.claude\.com)'
|
||||
|
||||
while IFS= read -r relpath; do
|
||||
[ -z "$relpath" ] && continue
|
||||
fpath="$PLUGIN_ROOT/$relpath"
|
||||
|
||||
if [ ! -f "$fpath" ]; then
|
||||
fail "Authoritative-claims file missing: $relpath"
|
||||
continue
|
||||
fi
|
||||
|
||||
HIT_COUNT="$(grep -cE "$ANTHROPIC_REGEX" "$fpath" || true)"
|
||||
if [ -z "$HIT_COUNT" ]; then HIT_COUNT=0; fi
|
||||
|
||||
if [ "$HIT_COUNT" -ge 1 ]; then
|
||||
pass "$relpath: $HIT_COUNT Anthropic-domain URL citation(s)"
|
||||
else
|
||||
fail "$relpath: zero Anthropic-domain URL citations (anthropic.com expected)"
|
||||
fi
|
||||
done < <(printf '%s\n' "$AUTH_FILES")
|
||||
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
printf "Pass: %d Fail: %d Warn: %d\n" "$PASS" "$FAIL" "$WARN"
|
||||
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
Loading…
Add table
Add a link
Reference in a new issue