feat(ms-ai-architect): G1 v3 bake-off MÅLT — v3 regredierte (P 92.1→89.7, R flat), v2 beholdt; G5 gull-friskhet åpnet [skip-docs]

scripts/kb-eval/data/judge-bakeoff-report-v3.json

@@ -0,0 +1,234 @@
+{
+  "_meta": {
+    "source": "gold-correctness-set.json + judge-bakeoff-results.json",
+    "thresholds": {
+      "minRecall": 0.7,
+      "minPrecision": 0.6
+    },
+    "judged": 255
+  },
+  "population": {
+    "total": 255,
+    "verifiable": 240,
+    "positives": 40,
+    "negatives": 200,
+    "unsourcedInP": 15
+  },
+  "arms": {
+    "staleness": {
+      "tp": 0,
+      "fp": 0,
+      "fn": 40,
+      "tn": 200,
+      "positives": 40,
+      "negatives": 200,
+      "flagged": 0,
+      "precision": null,
+      "recall": 0,
+      "f1": null,
+      "recallWilson": {
+        "p": 0,
+        "low": 0,
+        "high": 0.08762453925039232
+      },
+      "precisionWilson": null
+    },
+    "judge": {
+      "tp": 35,
+      "fp": 4,
+      "fn": 5,
+      "tn": 196,
+      "positives": 40,
+      "negatives": 200,
+      "flagged": 39,
+      "precision": 0.8974358974358975,
+      "recall": 0.875,
+      "f1": 0.8860759493670887,
+      "recallWilson": {
+        "p": 0.875,
+        "low": 0.7388757932976187,
+        "high": 0.9454058022645873
+      },
+      "precisionWilson": {
+        "p": 0.8974358974358975,
+        "low": 0.7642084129775517,
+        "high": 0.9593873444171301
+      }
+    },
+    "hybrid": {
+      "tp": 35,
+      "fp": 4,
+      "fn": 5,
+      "tn": 196,
+      "positives": 40,
+      "negatives": 200,
+      "flagged": 39,
+      "precision": 0.8974358974358975,
+      "recall": 0.875,
+      "f1": 0.8860759493670887,
+      "recallWilson": {
+        "p": 0.875,
+        "low": 0.7388757932976187,
+        "high": 0.9454058022645873
+      },
+      "precisionWilson": {
+        "p": 0.8974358974358975,
+        "low": 0.7642084129775517,
+        "high": 0.9593873444171301
+      }
+    }
+  },
+  "sourceSilent": {
+    "onVerifiableNegative": 0,
+    "onVerifiableError": 0,
+    "agreesWithUnsourced": 2,
+    "disagreesWithUnsourced": 13
+  },
+  "byClaimType": {
+    "version": {
+      "tp": 7,
+      "fp": 0,
+      "fn": 0,
+      "tn": 21,
+      "positives": 7,
+      "negatives": 21,
+      "flagged": 7,
+      "precision": 1,
+      "recall": 1,
+      "f1": 1,
+      "recallWilson": {
+        "p": 1,
+        "low": 0.6456611570247934,
+        "high": 1
+      },
+      "precisionWilson": {
+        "p": 1,
+        "low": 0.6456611570247934,
+        "high": 1
+      }
+    },
+    "tpm": {
+      "tp": 5,
+      "fp": 0,
+      "fn": 0,
+      "tn": 20,
+      "positives": 5,
+      "negatives": 20,
+      "flagged": 5,
+      "precision": 1,
+      "recall": 1,
+      "f1": 1,
+      "recallWilson": {
+        "p": 1,
+        "low": 0.5655085052479191,
+        "high": 1
+      },
+      "precisionWilson": {
+        "p": 1,
+        "low": 0.5655085052479191,
+        "high": 1
+      }
+    },
+    "region": {
+      "tp": 1,
+      "fp": 1,
+      "fn": 0,
+      "tn": 13,
+      "positives": 1,
+      "negatives": 14,
+      "flagged": 2,
+      "precision": 0.5,
+      "recall": 1,
+      "f1": 0.6666666666666666,
+      "recallWilson": {
+        "p": 1,
+        "low": 0.2065432914738929,
+        "high": 1
+      },
+      "precisionWilson": {
+        "p": 0.5,
+        "low": 0.09452865480086614,
+        "high": 0.9054713451991339
+      }
+    },
+    "status": {
+      "tp": 6,
+      "fp": 3,
+      "fn": 2,
+      "tn": 42,
+      "positives": 8,
+      "negatives": 45,
+      "flagged": 9,
+      "precision": 0.6666666666666666,
+      "recall": 0.75,
+      "f1": 0.7058823529411765,
+      "recallWilson": {
+        "p": 0.75,
+        "low": 0.40926987910258916,
+        "high": 0.9285223111419724
+      },
+      "precisionWilson": {
+        "p": 0.6666666666666666,
+        "low": 0.3541973474990897,
+        "high": 0.8794184013172571
+      }
+    },
+    "taxonomy": {
+      "tp": 9,
+      "fp": 0,
+      "fn": 2,
+      "tn": 86,
+      "positives": 11,
+      "negatives": 86,
+      "flagged": 9,
+      "precision": 1,
+      "recall": 0.8181818181818182,
+      "f1": 0.9,
+      "recallWilson": {
+        "p": 0.8181818181818182,
+        "low": 0.5230138624217553,
+        "high": 0.9486333993289995
+      },
+      "precisionWilson": {
+        "p": 1,
+        "low": 0.7008472464490407,
+        "high": 1
+      }
+    },
+    "sku": {
+      "tp": 7,
+      "fp": 0,
+      "fn": 1,
+      "tn": 14,
+      "positives": 8,
+      "negatives": 14,
+      "flagged": 7,
+      "precision": 1,
+      "recall": 0.875,
+      "f1": 0.9333333333333333,
+      "recallWilson": {
+        "p": 0.875,
+        "low": 0.5291051942301386,
+        "high": 0.9775830911367038
+      },
+      "precisionWilson": {
+        "p": 1,
+        "low": 0.6456611570247934,
+        "high": 1
+      }
+    }
+  },
+  "gate": {
+    "pass": true,
+    "recallOk": true,
+    "precisionOk": true,
+    "beatsStaleness": true,
+    "thresholds": {
+      "minRecall": 0.7,
+      "minPrecision": 0.6
+    },
+    "reasons": [
+      "all criteria met"
+    ]
+  }
+}

scripts/kb-eval/data/judge-bakeoff-report-v3.md

@@ -0,0 +1,54 @@
+# Judge bake-off-rapport — S1 (Fase 3 de-risk)
+
+_Generert deterministisk av `run-judge-bakeoff.mjs` over `gold-correctness-set.json` + `judge-bakeoff-results.json`. Tall fra testet `lib/judge-bakeoff.mjs`. Ikke rediger for hånd — regenerer._
+
+**Forhåndsregistrert gate (låst FØR fan-out):** recall ≥ 0.7, presisjon ≥ 0.6, OG judge-recall > staleness-recall.
+
+## Evaluerings-populasjon (P)
+
+Volatil stratum + fetchbare claim_types (price ekskludert) — der feilene bor; unngår «invertert leverage».
+
+| metrikk | verdi |
+|---|---|
+| P totalt | 255 |
+| Verifiserbare (correct/outdated/wrong) | 240 |
+| Positive (reelle feil å fange) | 40 |
+| Negative (correct) | 200 |
+| Unsourced i P (kjørt, men utenfor P/R) | 15 |
+
+## Arm-sammenligning (detektering over de 240 verifiserbare)
+
+| arm | TP | FP | FN | TN | presisjon | recall | recall Wilson 95% | F1 |
+|---|---|---|---|---|---|---|---|---|
+| staleness (billig baseline) | 0 | 0 | 40 | 200 | n/a | 0.0% | [0.0%, 8.8%] | n/a |
+| judge (per-påstand groundedness) | 35 | 4 | 5 | 196 | 89.7% | 87.5% | [73.9%, 94.5%] | 0.886 |
+| hybrid (union) | 35 | 4 | 5 | 196 | 89.7% | 87.5% | [73.9%, 94.5%] | 0.886 |
+
+## Judge per claim_type (verifiserbar delmengde)
+
+| claim_type | positive | TP | FP | FN | presisjon | recall |
+|---|---|---|---|---|---|---|
+| taxonomy | 11 | 9 | 0 | 2 | 100.0% | 81.8% |
+| status | 8 | 6 | 3 | 2 | 66.7% | 75.0% |
+| sku | 8 | 7 | 0 | 1 | 100.0% | 87.5% |
+| version | 7 | 7 | 0 | 0 | 100.0% | 100.0% |
+| tpm | 5 | 5 | 0 | 0 | 100.0% | 100.0% |
+| region | 1 | 1 | 1 | 0 | 50.0% | 100.0% |
+
+## source_silent-diagnostikk
+
+Judgen hentet siden men fant ikke verdien. Diagnostisk, ikke et flagg.
+
+| signal | antall | tolkning |
+|---|---|---|
+| På verifiserbar feil | 0 | judge-bom: reell feil oversett via «kan ikke verifisere» |
+| På verifiserbar correct | 0 | judge reproduserte ikke et korrekt faktum mennesket fant |
+| Enig med unsourced | 2 | judge reproduserer den uverifiserbare grensen (godt) |
+| Uenig med unsourced | 13 | judge hevdet grunnet/ugrunnet der mennesket ikke fant kilde |
+
+## GATE: ✅ PASS — bygg S3
+
+- recall 0.875 ≥ 0.7? **ja**
+- presisjon 0.897 ≥ 0.6? **ja**
+- slår staleness (recall 0.000)? **ja**
+- begrunnelse: all criteria met