test(ms-ai-architect): playground v3 parser fixture tests + run-e2e integration [skip-docs]
Step 15 (Wave 5). tests/test-playground-parsers.sh (new): - Iterates 17 expected fixtures (canonical archetype-routing list). - Validates each present + >= 20 lines + has section headers (## ). - Graceful-degrade: missing fixtures yield warn, not fail. - Greps 14 parser-function names + window.__PARSERS exposure. - Validates all 14 archetype routing keys in PARSERS object (aiact, requirements-list, text-document, fria, conformity-checklist, matrix-risk, matrix-risk-6x5, findings, cost-distribution, capability, phased-plan, markdown, verdict, comparison). - Validates handlePasteImport function + window.__handlePasteImport. - Bash 3.2-compatible. Result: 70/70 PASS. tests/run-e2e.sh (modify): - Adds --playground flag dispatching test-playground-v3.sh + test-playground-parsers.sh. - --all and no-arg invocation both include the new suite. Verify: bash tests/run-e2e.sh --playground -> exit 0 (170 + 70 PASS).
This commit is contained in:
parent
64441847f0
commit
e85f3fc9e9
2 changed files with 176 additions and 7 deletions
|
|
@ -1,6 +1,6 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# run-e2e.sh — Run E2E regression tests for agent outputs
|
# run-e2e.sh — Run E2E regression tests for agent outputs
|
||||||
# Usage: bash tests/run-e2e.sh [--security] [--cost] [--summary] [--all]
|
# Usage: bash tests/run-e2e.sh [--security] [--cost] [--summary] [--ros] [--ai-act] [--playground] [--all]
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
|
@ -16,6 +16,7 @@ RUN_COST=false
|
||||||
RUN_SUMMARY=false
|
RUN_SUMMARY=false
|
||||||
RUN_ROS=false
|
RUN_ROS=false
|
||||||
RUN_AI_ACT=false
|
RUN_AI_ACT=false
|
||||||
|
RUN_PLAYGROUND=false
|
||||||
|
|
||||||
if [ $# -eq 0 ] || [ "${1:-}" = "--all" ]; then
|
if [ $# -eq 0 ] || [ "${1:-}" = "--all" ]; then
|
||||||
RUN_SECURITY=true
|
RUN_SECURITY=true
|
||||||
|
|
@ -23,16 +24,18 @@ if [ $# -eq 0 ] || [ "${1:-}" = "--all" ]; then
|
||||||
RUN_SUMMARY=true
|
RUN_SUMMARY=true
|
||||||
RUN_ROS=true
|
RUN_ROS=true
|
||||||
RUN_AI_ACT=true
|
RUN_AI_ACT=true
|
||||||
|
RUN_PLAYGROUND=true
|
||||||
else
|
else
|
||||||
while [ $# -gt 0 ]; do
|
while [ $# -gt 0 ]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
--security) RUN_SECURITY=true ;;
|
--security) RUN_SECURITY=true ;;
|
||||||
--cost) RUN_COST=true ;;
|
--cost) RUN_COST=true ;;
|
||||||
--summary) RUN_SUMMARY=true ;;
|
--summary) RUN_SUMMARY=true ;;
|
||||||
--ros) RUN_ROS=true ;;
|
--ros) RUN_ROS=true ;;
|
||||||
--ai-act) RUN_AI_ACT=true ;;
|
--ai-act) RUN_AI_ACT=true ;;
|
||||||
|
--playground) RUN_PLAYGROUND=true ;;
|
||||||
*)
|
*)
|
||||||
echo "Usage: bash tests/run-e2e.sh [--security] [--cost] [--summary] [--ros] [--ai-act] [--all]"
|
echo "Usage: bash tests/run-e2e.sh [--security] [--cost] [--summary] [--ros] [--ai-act] [--playground] [--all]"
|
||||||
exit 1
|
exit 1
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
@ -67,6 +70,11 @@ if $RUN_AI_ACT; then
|
||||||
bash "$SCRIPT_DIR/test-ai-act-output.sh" || FAILURES=$((FAILURES + 1))
|
bash "$SCRIPT_DIR/test-ai-act-output.sh" || FAILURES=$((FAILURES + 1))
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if $RUN_PLAYGROUND; then
|
||||||
|
bash "$SCRIPT_DIR/test-playground-v3.sh" || FAILURES=$((FAILURES + 1))
|
||||||
|
bash "$SCRIPT_DIR/test-playground-parsers.sh" || FAILURES=$((FAILURES + 1))
|
||||||
|
fi
|
||||||
|
|
||||||
echo -e "${CYAN}══════════════════════════════════════════════${NC}"
|
echo -e "${CYAN}══════════════════════════════════════════════${NC}"
|
||||||
if [ "$FAILURES" -eq 0 ]; then
|
if [ "$FAILURES" -eq 0 ]; then
|
||||||
echo -e "${GREEN} All E2E suites passed${NC}"
|
echo -e "${GREEN} All E2E suites passed${NC}"
|
||||||
|
|
|
||||||
161
plugins/ms-ai-architect/tests/test-playground-parsers.sh
Executable file
161
plugins/ms-ai-architect/tests/test-playground-parsers.sh
Executable file
|
|
@ -0,0 +1,161 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# test-playground-parsers.sh — Parser-fixture-validering for Playground v3
|
||||||
|
#
|
||||||
|
# Itererer 17 forventede fixture-filer (kanonisk liste fra Step 10),
|
||||||
|
# bekrefter at de finnes og er >= 20 linjer, og bekrefter at de 14
|
||||||
|
# parser-funksjons-navnene fra archetype-routing-tabellen finnes i v3 HTML.
|
||||||
|
#
|
||||||
|
# Designet for graceful-degrade: hvis Step 10 har skipped capture, så
|
||||||
|
# kjører testen kun parser-name-grep og rapporterer pending fixtures
|
||||||
|
# som warn (ikke fail). Slik kan testen integreres tidlig.
|
||||||
|
#
|
||||||
|
# Bash 3.2-kompatibel.
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PLUGIN_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
HTML_FILE="$PLUGIN_ROOT/playground/ms-ai-architect-v3.html"
|
||||||
|
FIXTURES_DIR="$PLUGIN_ROOT/playground/test-fixtures"
|
||||||
|
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
source "$PLUGIN_ROOT/tests/lib/e2e-helpers.sh"
|
||||||
|
|
||||||
|
init_suite "Playground v3 — Parser fixtures"
|
||||||
|
|
||||||
|
# Forventede fixtures per kanonisk archetype-routing-tabell (17 stk).
|
||||||
|
# Rekkefølgen følger plan.md: 6 regulatory + 3 security + 2 economy + 6 documentation
|
||||||
|
EXPECTED_FIXTURES="classify requirements transparency frimpact conformity dpia security ros review cost license adr summary poc utredning compare migrate"
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# 1. Fixtures-katalog finnes
|
||||||
|
# -------------------------------------------------------
|
||||||
|
if [ ! -d "$FIXTURES_DIR" ]; then
|
||||||
|
fail "Fixture-katalog mangler: $FIXTURES_DIR"
|
||||||
|
print_summary
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
pass "Fixture-katalog eksisterer: $FIXTURES_DIR"
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# 2. Iterer 17 forventede fixtures, sjekk eksistens + min linjer
|
||||||
|
# -------------------------------------------------------
|
||||||
|
actual=0
|
||||||
|
expected_total=0
|
||||||
|
for cmd in $EXPECTED_FIXTURES; do
|
||||||
|
expected_total=$((expected_total + 1))
|
||||||
|
fixture="$FIXTURES_DIR/$cmd.md"
|
||||||
|
if [ -f "$fixture" ]; then
|
||||||
|
actual=$((actual + 1))
|
||||||
|
line_count=$(wc -l < "$fixture" | tr -d ' ')
|
||||||
|
if [ "${line_count:-0}" -ge 20 ]; then
|
||||||
|
pass "Fixture $cmd.md eksisterer og >= 20 linjer ($line_count)"
|
||||||
|
else
|
||||||
|
fail "Fixture $cmd.md eksisterer men < 20 linjer ($line_count)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# Capture-status
|
||||||
|
if [ "$actual" -eq 0 ]; then
|
||||||
|
warn "Ingen fixtures kapturet ennå (Step 10 skipped). Kjører kun parser-name-grep."
|
||||||
|
elif [ "$actual" -lt "$expected_total" ]; then
|
||||||
|
warn "$actual av $expected_total fixtures kapturet. Resterende rapporteres som pending."
|
||||||
|
pending=$((expected_total - actual))
|
||||||
|
for cmd in $EXPECTED_FIXTURES; do
|
||||||
|
[ -f "$FIXTURES_DIR/$cmd.md" ] && continue
|
||||||
|
warn "Fixture pending: $cmd.md"
|
||||||
|
done
|
||||||
|
pass "Capture-status: $actual/$expected_total fixtures (graceful-degrade tillatt)"
|
||||||
|
else
|
||||||
|
pass "Alle $expected_total fixtures kapturet"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# 3. Parser-funksjons-navn til stede i v3 HTML
|
||||||
|
# -------------------------------------------------------
|
||||||
|
if [ ! -f "$HTML_FILE" ]; then
|
||||||
|
fail "v3 HTML mangler: $HTML_FILE"
|
||||||
|
print_summary
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
pass "v3 HTML eksisterer: $HTML_FILE"
|
||||||
|
|
||||||
|
# 14 parsere fra kanonisk routing-tabell
|
||||||
|
PARSERS="parseAiAct parseRequirements parseTextDocument parseFria parseConformityChecklist parseMatrixRisk parseMatrixRisk6x5 parseFindings parseCostDistribution parseCapabilityMatrix parsePhasedPlan parseMarkdown parseVerdict parseComparison"
|
||||||
|
parser_hits=0
|
||||||
|
for p in $PARSERS; do
|
||||||
|
if grep -qE "function ${p}\b" "$HTML_FILE"; then
|
||||||
|
pass "Parser-funksjon definert: $p"
|
||||||
|
parser_hits=$((parser_hits + 1))
|
||||||
|
else
|
||||||
|
fail "Parser-funksjon mangler: $p"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ "$parser_hits" -eq 14 ]; then
|
||||||
|
pass "14/14 parsere til stede i v3 HTML"
|
||||||
|
else
|
||||||
|
fail "Forventet 14 parsere, fant $parser_hits"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# 4. PARSERS routing-objekt eksponert via window.__PARSERS
|
||||||
|
# -------------------------------------------------------
|
||||||
|
if grep -qE "window\.__PARSERS *=" "$HTML_FILE"; then
|
||||||
|
pass "window.__PARSERS eksponert (run-time tilgjengelig)"
|
||||||
|
else
|
||||||
|
fail "window.__PARSERS mangler"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Sjekk at hver canonical archetype er en nøkkel i PARSERS-objektet.
|
||||||
|
# Faktiske nøkler følger PARSERS-objektet i v3 HTML (ikke alle = report_archetype-verdier
|
||||||
|
# på CATALOG-commands; noen er normalisert: "aiact", "requirements-list", "capability").
|
||||||
|
ARCHETYPES="aiact requirements-list text-document fria conformity-checklist matrix-risk matrix-risk-6x5 findings cost-distribution capability phased-plan markdown verdict comparison"
|
||||||
|
arch_hits=0
|
||||||
|
for a in $ARCHETYPES; do
|
||||||
|
if grep -qE "['\"]${a}['\"] *: *parse" "$HTML_FILE"; then
|
||||||
|
pass "Archetype-routing-nøkkel: '${a}'"
|
||||||
|
arch_hits=$((arch_hits + 1))
|
||||||
|
else
|
||||||
|
fail "Archetype-routing-nøkkel mangler: '${a}'"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ "$arch_hits" -eq 14 ]; then
|
||||||
|
pass "14/14 archetype-nøkler i PARSERS-routing-objektet"
|
||||||
|
else
|
||||||
|
fail "Forventet 14 archetype-nøkler, fant $arch_hits"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# 5. Hver fixture indikerer korrekt arketype-treff (heuristisk)
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# Hvis fixture finnes og parser-name finnes, regn det som "rute-trygg".
|
||||||
|
for cmd in $EXPECTED_FIXTURES; do
|
||||||
|
fixture="$FIXTURES_DIR/$cmd.md"
|
||||||
|
[ -f "$fixture" ] || continue
|
||||||
|
# Sjekk at fixture-filen inneholder "## "-overskrift (alle parsere bruker section-split)
|
||||||
|
if grep -qE "^## " "$fixture"; then
|
||||||
|
pass "Fixture $cmd.md har section-headers (## )"
|
||||||
|
else
|
||||||
|
fail "Fixture $cmd.md mangler section-headers"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# 6. handlePasteImport routing-stub erstattet (Step 12 funksjonell)
|
||||||
|
# -------------------------------------------------------
|
||||||
|
if grep -qE "function handlePasteImport" "$HTML_FILE"; then
|
||||||
|
pass "handlePasteImport-funksjon definert"
|
||||||
|
else
|
||||||
|
fail "handlePasteImport-funksjon mangler"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if grep -qE "window\.__handlePasteImport *=" "$HTML_FILE"; then
|
||||||
|
pass "window.__handlePasteImport eksponert"
|
||||||
|
else
|
||||||
|
fail "window.__handlePasteImport mangler"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# -------------------------------------------------------
|
||||||
|
# Oppsummering
|
||||||
|
# -------------------------------------------------------
|
||||||
|
print_summary
|
||||||
Loading…
Add table
Add a link
Reference in a new issue