test(ms-ai-architect): playground v3 parser fixture tests + run-e2e integration [skip-docs]
Step 15 (Wave 5). tests/test-playground-parsers.sh (new): - Iterates 17 expected fixtures (canonical archetype-routing list). - Validates each present + >= 20 lines + has section headers (## ). - Graceful-degrade: missing fixtures yield warn, not fail. - Greps 14 parser-function names + window.__PARSERS exposure. - Validates all 14 archetype routing keys in PARSERS object (aiact, requirements-list, text-document, fria, conformity-checklist, matrix-risk, matrix-risk-6x5, findings, cost-distribution, capability, phased-plan, markdown, verdict, comparison). - Validates handlePasteImport function + window.__handlePasteImport. - Bash 3.2-compatible. Result: 70/70 PASS. tests/run-e2e.sh (modify): - Adds --playground flag dispatching test-playground-v3.sh + test-playground-parsers.sh. - --all and no-arg invocation both include the new suite. Verify: bash tests/run-e2e.sh --playground -> exit 0 (170 + 70 PASS).
This commit is contained in:
parent
64441847f0
commit
e85f3fc9e9
2 changed files with 176 additions and 7 deletions
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
# run-e2e.sh — Run E2E regression tests for agent outputs
|
||||
# Usage: bash tests/run-e2e.sh [--security] [--cost] [--summary] [--all]
|
||||
# Usage: bash tests/run-e2e.sh [--security] [--cost] [--summary] [--ros] [--ai-act] [--playground] [--all]
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
|
@ -16,6 +16,7 @@ RUN_COST=false
|
|||
RUN_SUMMARY=false
|
||||
RUN_ROS=false
|
||||
RUN_AI_ACT=false
|
||||
RUN_PLAYGROUND=false
|
||||
|
||||
if [ $# -eq 0 ] || [ "${1:-}" = "--all" ]; then
|
||||
RUN_SECURITY=true
|
||||
|
|
@ -23,16 +24,18 @@ if [ $# -eq 0 ] || [ "${1:-}" = "--all" ]; then
|
|||
RUN_SUMMARY=true
|
||||
RUN_ROS=true
|
||||
RUN_AI_ACT=true
|
||||
RUN_PLAYGROUND=true
|
||||
else
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--security) RUN_SECURITY=true ;;
|
||||
--cost) RUN_COST=true ;;
|
||||
--summary) RUN_SUMMARY=true ;;
|
||||
--ros) RUN_ROS=true ;;
|
||||
--ai-act) RUN_AI_ACT=true ;;
|
||||
--security) RUN_SECURITY=true ;;
|
||||
--cost) RUN_COST=true ;;
|
||||
--summary) RUN_SUMMARY=true ;;
|
||||
--ros) RUN_ROS=true ;;
|
||||
--ai-act) RUN_AI_ACT=true ;;
|
||||
--playground) RUN_PLAYGROUND=true ;;
|
||||
*)
|
||||
echo "Usage: bash tests/run-e2e.sh [--security] [--cost] [--summary] [--ros] [--ai-act] [--all]"
|
||||
echo "Usage: bash tests/run-e2e.sh [--security] [--cost] [--summary] [--ros] [--ai-act] [--playground] [--all]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
|
@ -67,6 +70,11 @@ if $RUN_AI_ACT; then
|
|||
bash "$SCRIPT_DIR/test-ai-act-output.sh" || FAILURES=$((FAILURES + 1))
|
||||
fi
|
||||
|
||||
if $RUN_PLAYGROUND; then
|
||||
bash "$SCRIPT_DIR/test-playground-v3.sh" || FAILURES=$((FAILURES + 1))
|
||||
bash "$SCRIPT_DIR/test-playground-parsers.sh" || FAILURES=$((FAILURES + 1))
|
||||
fi
|
||||
|
||||
echo -e "${CYAN}══════════════════════════════════════════════${NC}"
|
||||
if [ "$FAILURES" -eq 0 ]; then
|
||||
echo -e "${GREEN} All E2E suites passed${NC}"
|
||||
|
|
|
|||
161
plugins/ms-ai-architect/tests/test-playground-parsers.sh
Executable file
161
plugins/ms-ai-architect/tests/test-playground-parsers.sh
Executable file
|
|
@ -0,0 +1,161 @@
|
|||
#!/bin/bash
|
||||
# test-playground-parsers.sh — Parser-fixture-validering for Playground v3
|
||||
#
|
||||
# Itererer 17 forventede fixture-filer (kanonisk liste fra Step 10),
|
||||
# bekrefter at de finnes og er >= 20 linjer, og bekrefter at de 14
|
||||
# parser-funksjons-navnene fra archetype-routing-tabellen finnes i v3 HTML.
|
||||
#
|
||||
# Designet for graceful-degrade: hvis Step 10 har skipped capture, så
|
||||
# kjører testen kun parser-name-grep og rapporterer pending fixtures
|
||||
# som warn (ikke fail). Slik kan testen integreres tidlig.
|
||||
#
|
||||
# Bash 3.2-kompatibel.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PLUGIN_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
HTML_FILE="$PLUGIN_ROOT/playground/ms-ai-architect-v3.html"
|
||||
FIXTURES_DIR="$PLUGIN_ROOT/playground/test-fixtures"
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source "$PLUGIN_ROOT/tests/lib/e2e-helpers.sh"
|
||||
|
||||
init_suite "Playground v3 — Parser fixtures"
|
||||
|
||||
# Forventede fixtures per kanonisk archetype-routing-tabell (17 stk).
|
||||
# Rekkefølgen følger plan.md: 6 regulatory + 3 security + 2 economy + 6 documentation
|
||||
EXPECTED_FIXTURES="classify requirements transparency frimpact conformity dpia security ros review cost license adr summary poc utredning compare migrate"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# 1. Fixtures-katalog finnes
|
||||
# -------------------------------------------------------
|
||||
if [ ! -d "$FIXTURES_DIR" ]; then
|
||||
fail "Fixture-katalog mangler: $FIXTURES_DIR"
|
||||
print_summary
|
||||
exit 1
|
||||
fi
|
||||
pass "Fixture-katalog eksisterer: $FIXTURES_DIR"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# 2. Iterer 17 forventede fixtures, sjekk eksistens + min linjer
|
||||
# -------------------------------------------------------
|
||||
actual=0
|
||||
expected_total=0
|
||||
for cmd in $EXPECTED_FIXTURES; do
|
||||
expected_total=$((expected_total + 1))
|
||||
fixture="$FIXTURES_DIR/$cmd.md"
|
||||
if [ -f "$fixture" ]; then
|
||||
actual=$((actual + 1))
|
||||
line_count=$(wc -l < "$fixture" | tr -d ' ')
|
||||
if [ "${line_count:-0}" -ge 20 ]; then
|
||||
pass "Fixture $cmd.md eksisterer og >= 20 linjer ($line_count)"
|
||||
else
|
||||
fail "Fixture $cmd.md eksisterer men < 20 linjer ($line_count)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Capture-status
|
||||
if [ "$actual" -eq 0 ]; then
|
||||
warn "Ingen fixtures kapturet ennå (Step 10 skipped). Kjører kun parser-name-grep."
|
||||
elif [ "$actual" -lt "$expected_total" ]; then
|
||||
warn "$actual av $expected_total fixtures kapturet. Resterende rapporteres som pending."
|
||||
pending=$((expected_total - actual))
|
||||
for cmd in $EXPECTED_FIXTURES; do
|
||||
[ -f "$FIXTURES_DIR/$cmd.md" ] && continue
|
||||
warn "Fixture pending: $cmd.md"
|
||||
done
|
||||
pass "Capture-status: $actual/$expected_total fixtures (graceful-degrade tillatt)"
|
||||
else
|
||||
pass "Alle $expected_total fixtures kapturet"
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------
|
||||
# 3. Parser-funksjons-navn til stede i v3 HTML
|
||||
# -------------------------------------------------------
|
||||
if [ ! -f "$HTML_FILE" ]; then
|
||||
fail "v3 HTML mangler: $HTML_FILE"
|
||||
print_summary
|
||||
exit 1
|
||||
fi
|
||||
pass "v3 HTML eksisterer: $HTML_FILE"
|
||||
|
||||
# 14 parsere fra kanonisk routing-tabell
|
||||
PARSERS="parseAiAct parseRequirements parseTextDocument parseFria parseConformityChecklist parseMatrixRisk parseMatrixRisk6x5 parseFindings parseCostDistribution parseCapabilityMatrix parsePhasedPlan parseMarkdown parseVerdict parseComparison"
|
||||
parser_hits=0
|
||||
for p in $PARSERS; do
|
||||
if grep -qE "function ${p}\b" "$HTML_FILE"; then
|
||||
pass "Parser-funksjon definert: $p"
|
||||
parser_hits=$((parser_hits + 1))
|
||||
else
|
||||
fail "Parser-funksjon mangler: $p"
|
||||
fi
|
||||
done
|
||||
if [ "$parser_hits" -eq 14 ]; then
|
||||
pass "14/14 parsere til stede i v3 HTML"
|
||||
else
|
||||
fail "Forventet 14 parsere, fant $parser_hits"
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------
|
||||
# 4. PARSERS routing-objekt eksponert via window.__PARSERS
|
||||
# -------------------------------------------------------
|
||||
if grep -qE "window\.__PARSERS *=" "$HTML_FILE"; then
|
||||
pass "window.__PARSERS eksponert (run-time tilgjengelig)"
|
||||
else
|
||||
fail "window.__PARSERS mangler"
|
||||
fi
|
||||
|
||||
# Sjekk at hver canonical archetype er en nøkkel i PARSERS-objektet.
|
||||
# Faktiske nøkler følger PARSERS-objektet i v3 HTML (ikke alle = report_archetype-verdier
|
||||
# på CATALOG-commands; noen er normalisert: "aiact", "requirements-list", "capability").
|
||||
ARCHETYPES="aiact requirements-list text-document fria conformity-checklist matrix-risk matrix-risk-6x5 findings cost-distribution capability phased-plan markdown verdict comparison"
|
||||
arch_hits=0
|
||||
for a in $ARCHETYPES; do
|
||||
if grep -qE "['\"]${a}['\"] *: *parse" "$HTML_FILE"; then
|
||||
pass "Archetype-routing-nøkkel: '${a}'"
|
||||
arch_hits=$((arch_hits + 1))
|
||||
else
|
||||
fail "Archetype-routing-nøkkel mangler: '${a}'"
|
||||
fi
|
||||
done
|
||||
if [ "$arch_hits" -eq 14 ]; then
|
||||
pass "14/14 archetype-nøkler i PARSERS-routing-objektet"
|
||||
else
|
||||
fail "Forventet 14 archetype-nøkler, fant $arch_hits"
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------
|
||||
# 5. Hver fixture indikerer korrekt arketype-treff (heuristisk)
|
||||
# -------------------------------------------------------
|
||||
# Hvis fixture finnes og parser-name finnes, regn det som "rute-trygg".
|
||||
for cmd in $EXPECTED_FIXTURES; do
|
||||
fixture="$FIXTURES_DIR/$cmd.md"
|
||||
[ -f "$fixture" ] || continue
|
||||
# Sjekk at fixture-filen inneholder "## "-overskrift (alle parsere bruker section-split)
|
||||
if grep -qE "^## " "$fixture"; then
|
||||
pass "Fixture $cmd.md har section-headers (## )"
|
||||
else
|
||||
fail "Fixture $cmd.md mangler section-headers"
|
||||
fi
|
||||
done
|
||||
|
||||
# -------------------------------------------------------
|
||||
# 6. handlePasteImport routing-stub erstattet (Step 12 funksjonell)
|
||||
# -------------------------------------------------------
|
||||
if grep -qE "function handlePasteImport" "$HTML_FILE"; then
|
||||
pass "handlePasteImport-funksjon definert"
|
||||
else
|
||||
fail "handlePasteImport-funksjon mangler"
|
||||
fi
|
||||
|
||||
if grep -qE "window\.__handlePasteImport *=" "$HTML_FILE"; then
|
||||
pass "window.__handlePasteImport eksponert"
|
||||
else
|
||||
fail "window.__handlePasteImport mangler"
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Oppsummering
|
||||
# -------------------------------------------------------
|
||||
print_summary
|
||||
Loading…
Add table
Add a link
Reference in a new issue