#!/bin/bash # test-playground-fingerprints.sh — Playground v3 inferCommandIdFromMarkdown # # Verifiserer: # 1. PROJECT_VIEW_V2_BEGIN/END-markørene finnes # 2. inferCommandIdFromMarkdown matcher hver av 17 test-fixtures mot # egen commandId med confidence >= 0.6 (true-positive matrise) # 3. False-positive immunity: # - tom streng → null # - 100 tegn lorem ipsum → null # - kun "# Hello"-header → null # - mixed content med kun classify-header → matcher classify # 4. Sanity-asserts på COMMAND_FINGERPRINTS-shape og fingerprintScore-API # # Bash 3.2-kompatibel. Bruker node til JS-eval; ingen npm-deps. set -euo pipefail PLUGIN_ROOT="$(cd "$(dirname "$0")/.." && pwd)" HTML_FILE="$PLUGIN_ROOT/playground/ms-ai-architect-playground.html" FIXTURE_DIR="$PLUGIN_ROOT/playground/test-fixtures" # shellcheck disable=SC1091 source "$PLUGIN_ROOT/tests/lib/e2e-helpers.sh" init_suite "Playground v3 — inferCommandIdFromMarkdown fingerprints" # ---- 1. Filer eksisterer ---- if [ ! -f "$HTML_FILE" ]; then fail "HTML-fila finnes ikke: $HTML_FILE" print_summary; exit 1 fi pass "HTML-fil finnes: $(basename "$HTML_FILE")" if [ ! -d "$FIXTURE_DIR" ]; then fail "Fixture-mappe mangler: $FIXTURE_DIR" print_summary; exit 1 fi pass "Fixture-mappe finnes" # ---- 2. PROJECT_VIEW_V2-markører eksisterer ---- if grep -q "PROJECT_VIEW_V2_BEGIN" "$HTML_FILE" && grep -q "PROJECT_VIEW_V2_END" "$HTML_FILE"; then pass "PROJECT_VIEW_V2_BEGIN/END markører finnes" else fail "Mangler PROJECT_VIEW_V2-markører i HTML" print_summary; exit 1 fi # ---- 3. Kjør hele test-matrisen i én node-prosess (effektivt) ---- # Node-skriptet: # - ekstraherer PROJECT_VIEW_V2-blokken # - stubber dependencies # - leser alle 17 fixture-filer # - kjører true-positive + anti-match + sanity-tester # - skriver én linje per assert: "PASS|FAIL " NODE_OUT=$(node -e ' const fs = require("fs"); const path = require("path"); const htmlPath = process.argv[1]; const fixtureDir = process.argv[2]; const html = fs.readFileSync(htmlPath, "utf8"); const beginMarker = "// === PROJECT_VIEW_V2_BEGIN ==="; const endMarker = "// === PROJECT_VIEW_V2_END ==="; const beginIdx = html.indexOf(beginMarker); const endIdx = html.indexOf(endMarker); if (beginIdx < 0 || endIdx < 0) { console.error("MARKER_MISSING"); process.exit(2); } const block = html.substring(beginIdx, endIdx + endMarker.length); // 17 produces_report-renderers per PROJECT_VIEW_CONFIG.renderers. const RENDERER_IDS = ["renderAiActPyramid","renderRequirements","renderTransparency","renderFria","renderConformity","renderDpia","renderSecurity","renderRos","renderReview","renderCost","renderLicense","renderMigrate","renderAdr","renderSummary","renderPoc","renderUtredning","renderCompare"]; const rendererStubs = RENDERER_IDS.map(function (n) { return n + ": function () {}"; }).join(", "); // CATALOG.commands: 17 produces_report=true entries med id, label, category, renderer. const COMMANDS = [ { id: "classify", category: "regulatory", label: "EU AI Act — Klassifisering", renderer: "renderAiActPyramid", produces_report: true, report_archetype: "aiact" }, { id: "requirements", category: "regulatory", label: "EU AI Act — Krav per risiko", renderer: "renderRequirements", produces_report: true, report_archetype: "requirements-list" }, { id: "transparency", category: "regulatory", label: "Transparensnotis (Art. 13/50)", renderer: "renderTransparency", produces_report: true, report_archetype: "text-document" }, { id: "frimpact", category: "regulatory", label: "FRIA (Art. 27)", renderer: "renderFria", produces_report: true, report_archetype: "fria" }, { id: "conformity", category: "regulatory", label: "Samsvarsvurdering (Art. 43)", renderer: "renderConformity", produces_report: true, report_archetype: "conformity-checklist" }, { id: "dpia", category: "regulatory", label: "DPIA / PVK", renderer: "renderDpia", produces_report: true, report_archetype: "matrix-risk" }, { id: "security", category: "security", label: "Sikkerhetsvurdering (6×5)", renderer: "renderSecurity", produces_report: true, report_archetype: "matrix-risk-6x5" }, { id: "ros", category: "security", label: "ROS-analyse", renderer: "renderRos", produces_report: true, report_archetype: "matrix-risk" }, { id: "review", category: "security", label: "Arkitekturgjennomgang", renderer: "renderReview", produces_report: true, report_archetype: "findings" }, { id: "cost", category: "economy", label: "Kostnadsestimat", renderer: "renderCost", produces_report: true, report_archetype: "cost-distribution" }, { id: "license", category: "economy", label: "Lisenskartlegging", renderer: "renderLicense", produces_report: true, report_archetype: "scenario-comparison" }, { id: "migrate", category: "economy", label: "Migrasjonsplan", renderer: "renderMigrate", produces_report: true, report_archetype: "phase-plan" }, { id: "adr", category: "documentation", label: "ADR", renderer: "renderAdr", produces_report: true, report_archetype: "adr" }, { id: "summary", category: "documentation", label: "Beslutningsnotat", renderer: "renderSummary", produces_report: true, report_archetype: "verdict" }, { id: "poc", category: "documentation", label: "POC-plan", renderer: "renderPoc", produces_report: true, report_archetype: "phase-plan" }, { id: "utredning", category: "documentation", label: "Utredning", renderer: "renderUtredning", produces_report: true, report_archetype: "utredning" }, { id: "compare", category: "documentation", label: "Plattformsammenligning", renderer: "renderCompare", produces_report: true, report_archetype: "scenario-comparison" } ]; const stubs = ` const window = {}; function escapeHtml(s) { return String(s == null ? "" : s); } function escapeAttr(s) { return escapeHtml(s); } function renderPageShell(opts, body) { return "
" + (opts && opts.title || "") + "
" + (body || ""); } function renderVerdictPill(v) { return "" + v + ""; } function renderKeyStatsGrid(s) { return "
" + (s && s.length || 0) + "
"; } function inferVerdict() { return "n-a"; } function inferKeyStats() { return []; } const PARSERS = {}; const RENDERERS = { ` + rendererStubs + ` }; const CATALOG = { commands: ` + JSON.stringify(COMMANDS) + ` }; const ACTIONS = {}; const store = { state: { ui: {}, projects: [], activeProjectId: null }, save: function () {} }; function findProject() { return null; } function scheduleRender() {} `; const wrapped = stubs + block + "\nreturn { inferCommandIdFromMarkdown, fingerprintScore, COMMAND_FINGERPRINTS };"; let api; try { api = (new Function(wrapped))(); } catch (e) { console.error("EVAL_FAILED: " + e.message); process.exit(3); } function emit(ok, desc) { console.log((ok ? "PASS" : "FAIL") + "\t" + desc); } // ---- Sanity-asserts (5) ---- emit(typeof api.inferCommandIdFromMarkdown === "function", "inferCommandIdFromMarkdown er funksjon"); emit(typeof api.fingerprintScore === "function", "fingerprintScore er funksjon"); emit(typeof api.COMMAND_FINGERPRINTS === "object" && api.COMMAND_FINGERPRINTS !== null, "COMMAND_FINGERPRINTS er objekt"); const cfKeys = Object.keys(api.COMMAND_FINGERPRINTS); emit(cfKeys.length === 17, "COMMAND_FINGERPRINTS har 17 entries (got " + cfKeys.length + ")"); let allShapeOk = true; for (const k of cfKeys) { const v = api.COMMAND_FINGERPRINTS[k]; if (!v || !Array.isArray(v.headers) || !Array.isArray(v.keywords)) { allShapeOk = false; break; } } emit(allShapeOk, "Hver fingerprint har headers[] og keywords[]"); // ---- True-positive matrise (17 fixtures) ---- // // Kjente fingerprint/fixture-gap (oppdaget av denne testen, fix i sesjon 5): // - requirements.md har header "# EU AI Act — Krav for høyrisiko" som ikke // matcher /^\s*#\s*(AI\s*Act-?krav|Krav per|Requirements)/i. Annex IV- // omtale i tabellen gjør at conformity vinner med 0.76. // - license.md har header "# Lisens-kapabilitetsmatrise" som ikke matcher // /^\s*#\s*(Lisens(kart)?legging|License\s*Mapping)/i. // v1.15.0 (sesjon 5): begge gap-er lukket — requirements.headers og // license.headers utvidet i COMMAND_FINGERPRINTS. KNOWN_GAP_FIXTURES tømt. const KNOWN_GAP_FIXTURES = {}; const expectedIds = ["classify","requirements","transparency","frimpact","conformity","dpia","security","ros","review","cost","license","migrate","adr","summary","poc","utredning","compare"]; for (const cid of expectedIds) { const fxPath = path.join(fixtureDir, cid + ".md"); let text = ""; try { text = fs.readFileSync(fxPath, "utf8"); } catch (e) { emit(false, "fixture " + cid + ".md kunne ikke leses: " + e.message); continue; } const r = api.inferCommandIdFromMarkdown(text, {}); const matchedCid = r && r.commandId; const conf = r && r.confidence; const ok = r && r.commandId === cid && r.confidence >= 0.6; if (ok) { emit(true, "fixture " + cid + ".md → " + matchedCid + " conf=" + conf.toFixed(2)); } else if (KNOWN_GAP_FIXTURES[cid]) { console.log("WARN\tfixture " + cid + ".md → " + (matchedCid || "null") + " (KNOWN_GAP — fingerprint dekker ikke fixture-header; fix i sesjon 5)"); } else { emit(false, "fixture " + cid + ".md → " + (matchedCid || "null") + (conf != null ? " conf=" + conf.toFixed(2) : "")); } } // ---- Anti-match (4) ---- emit(api.inferCommandIdFromMarkdown("", {}) === null, "tom streng → null"); emit(api.inferCommandIdFromMarkdown(null, {}) === null, "null input → null"); const lorem = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim"; emit(api.inferCommandIdFromMarkdown(lorem, {}) === null, "100 tegn lorem ipsum → null (got " + JSON.stringify(api.inferCommandIdFromMarkdown(lorem, {})) + ")"); emit(api.inferCommandIdFromMarkdown("# Hello\n\nIngen relevant tekst.", {}) === null, "kun \"# Hello\"-header → null"); // ---- Mixed content: dominant header vinner ---- const mixed = "# EU AI Act — Klassifisering\n\nDette inneholder også owasp og prompt injection-referanser men headeren er klassifisering."; const mixedR = api.inferCommandIdFromMarkdown(mixed, {}); emit(mixedR !== null && mixedR.commandId === "classify", "mixed content med classify-header → " + (mixedR && mixedR.commandId)); // ---- fingerprintScore-API direkte ---- emit(api.fingerprintScore("any text", null) === 0, "fingerprintScore(null spec) === 0"); emit(api.fingerprintScore("", api.COMMAND_FINGERPRINTS.classify) === 0, "fingerprintScore(tom tekst) === 0"); ' "$HTML_FILE" "$FIXTURE_DIR" 2>&1) || NODE_RC=$? if [ "${NODE_RC:-0}" -ne 0 ]; then fail "node-eval feilet (rc=${NODE_RC:-0}): $NODE_OUT" print_summary; exit 1 fi # Parse PASS/FAIL/WARN-linjer fra node-output while IFS=$'\t' read -r status desc; do case "$status" in PASS) pass "$desc" ;; FAIL) fail "$desc" ;; WARN) warn "$desc" ;; esac done <<< "$NODE_OUT" print_summary