ktg-plugin-marketplace/plugins/ms-ai-architect/tests/test-playground-fingerprints.sh

#!/bin/bash
# test-playground-fingerprints.sh — Playground v3 inferCommandIdFromMarkdown
#
# Verifiserer:
#   1. PROJECT_VIEW_V2_BEGIN/END-markørene finnes
#   2. inferCommandIdFromMarkdown matcher hver av 17 test-fixtures mot
#      egen commandId med confidence >= 0.6 (true-positive matrise)
#   3. False-positive immunity:
#        - tom streng → null
#        - 100 tegn lorem ipsum → null
#        - kun "# Hello"-header → null
#        - mixed content med kun classify-header → matcher classify
#   4. Sanity-asserts på COMMAND_FINGERPRINTS-shape og fingerprintScore-API
#
# Bash 3.2-kompatibel. Bruker node til JS-eval; ingen npm-deps.

set -euo pipefail

PLUGIN_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
HTML_FILE="$PLUGIN_ROOT/playground/ms-ai-architect-playground.html"
FIXTURE_DIR="$PLUGIN_ROOT/playground/test-fixtures"

# shellcheck disable=SC1091
source "$PLUGIN_ROOT/tests/lib/e2e-helpers.sh"

init_suite "Playground v3 — inferCommandIdFromMarkdown fingerprints"

# ---- 1. Filer eksisterer ----
if [ ! -f "$HTML_FILE" ]; then
  fail "HTML-fila finnes ikke: $HTML_FILE"
  print_summary; exit 1
fi
pass "HTML-fil finnes: $(basename "$HTML_FILE")"

if [ ! -d "$FIXTURE_DIR" ]; then
  fail "Fixture-mappe mangler: $FIXTURE_DIR"
  print_summary; exit 1
fi
pass "Fixture-mappe finnes"

# ---- 2. PROJECT_VIEW_V2-markører eksisterer ----
if grep -q "PROJECT_VIEW_V2_BEGIN" "$HTML_FILE" && grep -q "PROJECT_VIEW_V2_END" "$HTML_FILE"; then
  pass "PROJECT_VIEW_V2_BEGIN/END markører finnes"
else
  fail "Mangler PROJECT_VIEW_V2-markører i HTML"
  print_summary; exit 1
fi

# ---- 3. Kjør hele test-matrisen i én node-prosess (effektivt) ----
# Node-skriptet:
#   - ekstraherer PROJECT_VIEW_V2-blokken
#   - stubber dependencies
#   - leser alle 17 fixture-filer
#   - kjører true-positive + anti-match + sanity-tester
#   - skriver én linje per assert: "PASS|FAIL <description>"

NODE_OUT=$(node -e '
const fs = require("fs");
const path = require("path");
const htmlPath = process.argv[1];
const fixtureDir = process.argv[2];

const html = fs.readFileSync(htmlPath, "utf8");
const beginMarker = "// === PROJECT_VIEW_V2_BEGIN ===";
const endMarker = "// === PROJECT_VIEW_V2_END ===";
const beginIdx = html.indexOf(beginMarker);
const endIdx = html.indexOf(endMarker);
if (beginIdx < 0 || endIdx < 0) {
  console.error("MARKER_MISSING");
  process.exit(2);
}
const block = html.substring(beginIdx, endIdx + endMarker.length);

// 17 produces_report-renderers per PROJECT_VIEW_CONFIG.renderers.
const RENDERER_IDS = ["renderAiActPyramid","renderRequirements","renderTransparency","renderFria","renderConformity","renderDpia","renderSecurity","renderRos","renderReview","renderCost","renderLicense","renderMigrate","renderAdr","renderSummary","renderPoc","renderUtredning","renderCompare"];
const rendererStubs = RENDERER_IDS.map(function (n) { return n + ": function () {}"; }).join(", ");

// CATALOG.commands: 17 produces_report=true entries med id, label, category, renderer.
const COMMANDS = [
  { id: "classify",     category: "regulatory",    label: "EU AI Act — Klassifisering",      renderer: "renderAiActPyramid", produces_report: true, report_archetype: "aiact" },
  { id: "requirements", category: "regulatory",    label: "EU AI Act — Krav per risiko",     renderer: "renderRequirements", produces_report: true, report_archetype: "requirements-list" },
  { id: "transparency", category: "regulatory",    label: "Transparensnotis (Art. 13/50)",   renderer: "renderTransparency", produces_report: true, report_archetype: "text-document" },
  { id: "frimpact",     category: "regulatory",    label: "FRIA (Art. 27)",                   renderer: "renderFria",         produces_report: true, report_archetype: "fria" },
  { id: "conformity",   category: "regulatory",    label: "Samsvarsvurdering (Art. 43)",      renderer: "renderConformity",   produces_report: true, report_archetype: "conformity-checklist" },
  { id: "dpia",         category: "regulatory",    label: "DPIA / PVK",                       renderer: "renderDpia",         produces_report: true, report_archetype: "matrix-risk" },
  { id: "security",     category: "security",      label: "Sikkerhetsvurdering (6×5)",        renderer: "renderSecurity",     produces_report: true, report_archetype: "matrix-risk-6x5" },
  { id: "ros",          category: "security",      label: "ROS-analyse",                      renderer: "renderRos",          produces_report: true, report_archetype: "matrix-risk" },
  { id: "review",       category: "security",      label: "Arkitekturgjennomgang",            renderer: "renderReview",       produces_report: true, report_archetype: "findings" },
  { id: "cost",         category: "economy",       label: "Kostnadsestimat",                  renderer: "renderCost",         produces_report: true, report_archetype: "cost-distribution" },
  { id: "license",      category: "economy",       label: "Lisenskartlegging",                renderer: "renderLicense",      produces_report: true, report_archetype: "scenario-comparison" },
  { id: "migrate",      category: "economy",       label: "Migrasjonsplan",                   renderer: "renderMigrate",      produces_report: true, report_archetype: "phase-plan" },
  { id: "adr",          category: "documentation", label: "ADR",                              renderer: "renderAdr",          produces_report: true, report_archetype: "adr" },
  { id: "summary",      category: "documentation", label: "Beslutningsnotat",                 renderer: "renderSummary",      produces_report: true, report_archetype: "verdict" },
  { id: "poc",          category: "documentation", label: "POC-plan",                         renderer: "renderPoc",          produces_report: true, report_archetype: "phase-plan" },
  { id: "utredning",    category: "documentation", label: "Utredning",                        renderer: "renderUtredning",    produces_report: true, report_archetype: "utredning" },
  { id: "compare",      category: "documentation", label: "Plattformsammenligning",           renderer: "renderCompare",      produces_report: true, report_archetype: "scenario-comparison" }
];

const stubs = `
  const window = {};
  function escapeHtml(s) { return String(s == null ? "" : s); }
  function escapeAttr(s) { return escapeHtml(s); }
  function renderPageShell(opts, body) { return "<header>" + (opts && opts.title || "") + "</header>" + (body || ""); }
  function renderVerdictPill(v) { return "<span class=\\"verdict-pill\\" data-verdict=\\"" + v + "\\">" + v + "</span>"; }
  function renderKeyStatsGrid(s) { return "<div class=\\"key-stats\\">" + (s && s.length || 0) + "</div>"; }
  function inferVerdict() { return "n-a"; }
  function inferKeyStats() { return []; }
  const PARSERS = {};
  const RENDERERS = { ` + rendererStubs + ` };
  const CATALOG = { commands: ` + JSON.stringify(COMMANDS) + ` };
  const ACTIONS = {};
  const store = { state: { ui: {}, projects: [], activeProjectId: null }, save: function () {} };
  function findProject() { return null; }
  function scheduleRender() {}
`;

const wrapped = stubs + block + "\nreturn { inferCommandIdFromMarkdown, fingerprintScore, COMMAND_FINGERPRINTS };";
let api;
try {
  api = (new Function(wrapped))();
} catch (e) {
  console.error("EVAL_FAILED: " + e.message);
  process.exit(3);
}

function emit(ok, desc) {
  console.log((ok ? "PASS" : "FAIL") + "\t" + desc);
}

// ---- Sanity-asserts (5) ----
emit(typeof api.inferCommandIdFromMarkdown === "function", "inferCommandIdFromMarkdown er funksjon");
emit(typeof api.fingerprintScore === "function", "fingerprintScore er funksjon");
emit(typeof api.COMMAND_FINGERPRINTS === "object" && api.COMMAND_FINGERPRINTS !== null, "COMMAND_FINGERPRINTS er objekt");
const cfKeys = Object.keys(api.COMMAND_FINGERPRINTS);
emit(cfKeys.length === 17, "COMMAND_FINGERPRINTS har 17 entries (got " + cfKeys.length + ")");
let allShapeOk = true;
for (const k of cfKeys) {
  const v = api.COMMAND_FINGERPRINTS[k];
  if (!v || !Array.isArray(v.headers) || !Array.isArray(v.keywords)) {
    allShapeOk = false; break;
  }
}
emit(allShapeOk, "Hver fingerprint har headers[] og keywords[]");

// ---- True-positive matrise (17 fixtures) ----
//
// Kjente fingerprint/fixture-gap (oppdaget av denne testen, fix i sesjon 5):
//   - requirements.md har header "# EU AI Act — Krav for høyrisiko" som ikke
//     matcher /^\s*#\s*(AI\s*Act-?krav|Krav per|Requirements)/i. Annex IV-
//     omtale i tabellen gjør at conformity vinner med 0.76.
//   - license.md har header "# Lisens-kapabilitetsmatrise" som ikke matcher
//     /^\s*#\s*(Lisens(kart)?legging|License\s*Mapping)/i.
// v1.15.0 (sesjon 5): begge gap-er lukket — requirements.headers og
// license.headers utvidet i COMMAND_FINGERPRINTS. KNOWN_GAP_FIXTURES tømt.
const KNOWN_GAP_FIXTURES = {};

const expectedIds = ["classify","requirements","transparency","frimpact","conformity","dpia","security","ros","review","cost","license","migrate","adr","summary","poc","utredning","compare"];
for (const cid of expectedIds) {
  const fxPath = path.join(fixtureDir, cid + ".md");
  let text = "";
  try { text = fs.readFileSync(fxPath, "utf8"); } catch (e) {
    emit(false, "fixture " + cid + ".md kunne ikke leses: " + e.message);
    continue;
  }
  const r = api.inferCommandIdFromMarkdown(text, {});
  const matchedCid = r && r.commandId;
  const conf = r && r.confidence;
  const ok = r && r.commandId === cid && r.confidence >= 0.6;
  if (ok) {
    emit(true, "fixture " + cid + ".md → " + matchedCid + " conf=" + conf.toFixed(2));
  } else if (KNOWN_GAP_FIXTURES[cid]) {
    console.log("WARN\tfixture " + cid + ".md → " + (matchedCid || "null") +
                " (KNOWN_GAP — fingerprint dekker ikke fixture-header; fix i sesjon 5)");
  } else {
    emit(false, "fixture " + cid + ".md → " + (matchedCid || "null") +
                (conf != null ? " conf=" + conf.toFixed(2) : ""));
  }
}

// ---- Anti-match (4) ----
emit(api.inferCommandIdFromMarkdown("", {}) === null,
     "tom streng → null");
emit(api.inferCommandIdFromMarkdown(null, {}) === null,
     "null input → null");
const lorem = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim";
emit(api.inferCommandIdFromMarkdown(lorem, {}) === null,
     "100 tegn lorem ipsum → null (got " + JSON.stringify(api.inferCommandIdFromMarkdown(lorem, {})) + ")");
emit(api.inferCommandIdFromMarkdown("# Hello\n\nIngen relevant tekst.", {}) === null,
     "kun \"# Hello\"-header → null");

// ---- Mixed content: dominant header vinner ----
const mixed = "# EU AI Act — Klassifisering\n\nDette inneholder også owasp og prompt injection-referanser men headeren er klassifisering.";
const mixedR = api.inferCommandIdFromMarkdown(mixed, {});
emit(mixedR !== null && mixedR.commandId === "classify",
     "mixed content med classify-header → " + (mixedR && mixedR.commandId));

// ---- fingerprintScore-API direkte ----
emit(api.fingerprintScore("any text", null) === 0,
     "fingerprintScore(null spec) === 0");
emit(api.fingerprintScore("", api.COMMAND_FINGERPRINTS.classify) === 0,
     "fingerprintScore(tom tekst) === 0");
' "$HTML_FILE" "$FIXTURE_DIR" 2>&1) || NODE_RC=$?

if [ "${NODE_RC:-0}" -ne 0 ]; then
  fail "node-eval feilet (rc=${NODE_RC:-0}): $NODE_OUT"
  print_summary; exit 1
fi

# Parse PASS/FAIL/WARN-linjer fra node-output
while IFS=$'\t' read -r status desc; do
  case "$status" in
    PASS) pass "$desc" ;;
    FAIL) fail "$desc" ;;
    WARN) warn "$desc" ;;
  esac
done <<< "$NODE_OUT"

print_summary