feat(ms-ai-architect): S1 v2 targeted iteration — GATE PASS (recall 84.2%, precision 84.2%) [skip-docs]
Operatørvalg (c): én målrettet, prinsipiell prompt-iterasjon på den diagnostiserte grounded-men-feil-feilmoden (eksakt-verdi-entailment). Terskel uendret; v1 frosset. Full blind v2 fan-out (15 batcher Opus 4.8 xhigh, 255 P-påstander dekket): - judge v2: recall 84.2% (32/38, PASS >=0.80, Wilson [69.6-92.6%]), presisjon 84.2% (PASS >=0.70), F1 0.842, slår staleness 0/38. - Fiksen løste målet: sku recall 37.5%->75.0%, taxonomy 66.7%->100%. - +6 ekte fangster (26->32) uten netto nye FP (6->6) => recall OG presisjon opp. Forbehold (ærlig): andre måling på samme frosne sett etter v1 (erkjent); Wilson nedre grense 69.6% < 0.80 ved n=38; én iterasjon. Gate-logikk => vei mot S3. Stoppet for operatør-beslutning (S2/S3), eskalerer ikke selv. run-judge-bakeoff.mjs: --results/--report-prefix flagg (v2 uten å klobbe v1). Suite 552/552.
This commit is contained in:
parent
79e34312ea
commit
4cd290c14b
6 changed files with 1436 additions and 3 deletions
234
scripts/kb-eval/data/judge-bakeoff-report-v2.json
Normal file
234
scripts/kb-eval/data/judge-bakeoff-report-v2.json
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
{
|
||||
"_meta": {
|
||||
"source": "gold-correctness-set.json + judge-bakeoff-results.json",
|
||||
"thresholds": {
|
||||
"minRecall": 0.8,
|
||||
"minPrecision": 0.7
|
||||
},
|
||||
"judged": 255
|
||||
},
|
||||
"population": {
|
||||
"total": 255,
|
||||
"verifiable": 240,
|
||||
"positives": 38,
|
||||
"negatives": 202,
|
||||
"unsourcedInP": 15
|
||||
},
|
||||
"arms": {
|
||||
"staleness": {
|
||||
"tp": 0,
|
||||
"fp": 0,
|
||||
"fn": 38,
|
||||
"tn": 202,
|
||||
"positives": 38,
|
||||
"negatives": 202,
|
||||
"flagged": 0,
|
||||
"precision": null,
|
||||
"recall": 0,
|
||||
"f1": null,
|
||||
"recallWilson": {
|
||||
"p": 0,
|
||||
"low": 0,
|
||||
"high": 0.09181293258383999
|
||||
},
|
||||
"precisionWilson": null
|
||||
},
|
||||
"judge": {
|
||||
"tp": 32,
|
||||
"fp": 6,
|
||||
"fn": 6,
|
||||
"tn": 196,
|
||||
"positives": 38,
|
||||
"negatives": 202,
|
||||
"flagged": 38,
|
||||
"precision": 0.8421052631578947,
|
||||
"recall": 0.8421052631578947,
|
||||
"f1": 0.8421052631578947,
|
||||
"recallWilson": {
|
||||
"p": 0.8421052631578947,
|
||||
"low": 0.6958287736272311,
|
||||
"high": 0.9255623777627731
|
||||
},
|
||||
"precisionWilson": {
|
||||
"p": 0.8421052631578947,
|
||||
"low": 0.6958287736272311,
|
||||
"high": 0.9255623777627731
|
||||
}
|
||||
},
|
||||
"hybrid": {
|
||||
"tp": 32,
|
||||
"fp": 6,
|
||||
"fn": 6,
|
||||
"tn": 196,
|
||||
"positives": 38,
|
||||
"negatives": 202,
|
||||
"flagged": 38,
|
||||
"precision": 0.8421052631578947,
|
||||
"recall": 0.8421052631578947,
|
||||
"f1": 0.8421052631578947,
|
||||
"recallWilson": {
|
||||
"p": 0.8421052631578947,
|
||||
"low": 0.6958287736272311,
|
||||
"high": 0.9255623777627731
|
||||
},
|
||||
"precisionWilson": {
|
||||
"p": 0.8421052631578947,
|
||||
"low": 0.6958287736272311,
|
||||
"high": 0.9255623777627731
|
||||
}
|
||||
}
|
||||
},
|
||||
"sourceSilent": {
|
||||
"onVerifiableNegative": 3,
|
||||
"onVerifiableError": 2,
|
||||
"agreesWithUnsourced": 5,
|
||||
"disagreesWithUnsourced": 10
|
||||
},
|
||||
"byClaimType": {
|
||||
"version": {
|
||||
"tp": 6,
|
||||
"fp": 0,
|
||||
"fn": 1,
|
||||
"tn": 21,
|
||||
"positives": 7,
|
||||
"negatives": 21,
|
||||
"flagged": 6,
|
||||
"precision": 1,
|
||||
"recall": 0.8571428571428571,
|
||||
"f1": 0.923076923076923,
|
||||
"recallWilson": {
|
||||
"p": 0.8571428571428571,
|
||||
"low": 0.4868654966809701,
|
||||
"high": 0.9743210440510252
|
||||
},
|
||||
"precisionWilson": {
|
||||
"p": 1,
|
||||
"low": 0.6096569663469354,
|
||||
"high": 0.9999999999999999
|
||||
}
|
||||
},
|
||||
"tpm": {
|
||||
"tp": 4,
|
||||
"fp": 0,
|
||||
"fn": 1,
|
||||
"tn": 20,
|
||||
"positives": 5,
|
||||
"negatives": 20,
|
||||
"flagged": 4,
|
||||
"precision": 1,
|
||||
"recall": 0.8,
|
||||
"f1": 0.888888888888889,
|
||||
"recallWilson": {
|
||||
"p": 0.8,
|
||||
"low": 0.3755282641185388,
|
||||
"high": 0.9637768390302125
|
||||
},
|
||||
"precisionWilson": {
|
||||
"p": 1,
|
||||
"low": 0.5100999795960008,
|
||||
"high": 1
|
||||
}
|
||||
},
|
||||
"region": {
|
||||
"tp": 1,
|
||||
"fp": 0,
|
||||
"fn": 1,
|
||||
"tn": 13,
|
||||
"positives": 2,
|
||||
"negatives": 13,
|
||||
"flagged": 1,
|
||||
"precision": 1,
|
||||
"recall": 0.5,
|
||||
"f1": 0.6666666666666666,
|
||||
"recallWilson": {
|
||||
"p": 0.5,
|
||||
"low": 0.09452865480086614,
|
||||
"high": 0.9054713451991339
|
||||
},
|
||||
"precisionWilson": {
|
||||
"p": 1,
|
||||
"low": 0.2065432914738929,
|
||||
"high": 1
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"tp": 6,
|
||||
"fp": 1,
|
||||
"fn": 1,
|
||||
"tn": 45,
|
||||
"positives": 7,
|
||||
"negatives": 46,
|
||||
"flagged": 7,
|
||||
"precision": 0.8571428571428571,
|
||||
"recall": 0.8571428571428571,
|
||||
"f1": 0.8571428571428571,
|
||||
"recallWilson": {
|
||||
"p": 0.8571428571428571,
|
||||
"low": 0.4868654966809701,
|
||||
"high": 0.9743210440510252
|
||||
},
|
||||
"precisionWilson": {
|
||||
"p": 0.8571428571428571,
|
||||
"low": 0.4868654966809701,
|
||||
"high": 0.9743210440510252
|
||||
}
|
||||
},
|
||||
"taxonomy": {
|
||||
"tp": 9,
|
||||
"fp": 5,
|
||||
"fn": 0,
|
||||
"tn": 83,
|
||||
"positives": 9,
|
||||
"negatives": 88,
|
||||
"flagged": 14,
|
||||
"precision": 0.6428571428571429,
|
||||
"recall": 1,
|
||||
"f1": 0.782608695652174,
|
||||
"recallWilson": {
|
||||
"p": 1,
|
||||
"low": 0.7008472464490407,
|
||||
"high": 1
|
||||
},
|
||||
"precisionWilson": {
|
||||
"p": 0.6428571428571429,
|
||||
"low": 0.3876400468214041,
|
||||
"high": 0.8365550926279728
|
||||
}
|
||||
},
|
||||
"sku": {
|
||||
"tp": 6,
|
||||
"fp": 0,
|
||||
"fn": 2,
|
||||
"tn": 14,
|
||||
"positives": 8,
|
||||
"negatives": 14,
|
||||
"flagged": 6,
|
||||
"precision": 1,
|
||||
"recall": 0.75,
|
||||
"f1": 0.8571428571428571,
|
||||
"recallWilson": {
|
||||
"p": 0.75,
|
||||
"low": 0.40926987910258916,
|
||||
"high": 0.9285223111419724
|
||||
},
|
||||
"precisionWilson": {
|
||||
"p": 1,
|
||||
"low": 0.6096569663469354,
|
||||
"high": 0.9999999999999999
|
||||
}
|
||||
}
|
||||
},
|
||||
"gate": {
|
||||
"pass": true,
|
||||
"recallOk": true,
|
||||
"precisionOk": true,
|
||||
"beatsStaleness": true,
|
||||
"thresholds": {
|
||||
"minRecall": 0.8,
|
||||
"minPrecision": 0.7
|
||||
},
|
||||
"reasons": [
|
||||
"all criteria met"
|
||||
]
|
||||
}
|
||||
}
|
||||
54
scripts/kb-eval/data/judge-bakeoff-report-v2.md
Normal file
54
scripts/kb-eval/data/judge-bakeoff-report-v2.md
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
# Judge bake-off-rapport — S1 (Fase 3 de-risk)
|
||||
|
||||
_Generert deterministisk av `run-judge-bakeoff.mjs` over `gold-correctness-set.json` + `judge-bakeoff-results.json`. Tall fra testet `lib/judge-bakeoff.mjs`. Ikke rediger for hånd — regenerer._
|
||||
|
||||
**Forhåndsregistrert gate (låst FØR fan-out):** recall ≥ 0.8, presisjon ≥ 0.7, OG judge-recall > staleness-recall.
|
||||
|
||||
## Evaluerings-populasjon (P)
|
||||
|
||||
Volatil stratum + fetchbare claim_types (price ekskludert) — der feilene bor; unngår «invertert leverage».
|
||||
|
||||
| metrikk | verdi |
|
||||
|---|---|
|
||||
| P totalt | 255 |
|
||||
| Verifiserbare (correct/outdated/wrong) | 240 |
|
||||
| Positive (reelle feil å fange) | 38 |
|
||||
| Negative (correct) | 202 |
|
||||
| Unsourced i P (kjørt, men utenfor P/R) | 15 |
|
||||
|
||||
## Arm-sammenligning (detektering over de 240 verifiserbare)
|
||||
|
||||
| arm | TP | FP | FN | TN | presisjon | recall | recall Wilson 95% | F1 |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| staleness (billig baseline) | 0 | 0 | 38 | 202 | n/a | 0.0% | [0.0%, 9.2%] | n/a |
|
||||
| judge (per-påstand groundedness) | 32 | 6 | 6 | 196 | 84.2% | 84.2% | [69.6%, 92.6%] | 0.842 |
|
||||
| hybrid (union) | 32 | 6 | 6 | 196 | 84.2% | 84.2% | [69.6%, 92.6%] | 0.842 |
|
||||
|
||||
## Judge per claim_type (verifiserbar delmengde)
|
||||
|
||||
| claim_type | positive | TP | FP | FN | presisjon | recall |
|
||||
|---|---|---|---|---|---|---|
|
||||
| taxonomy | 9 | 9 | 5 | 0 | 64.3% | 100.0% |
|
||||
| sku | 8 | 6 | 0 | 2 | 100.0% | 75.0% |
|
||||
| version | 7 | 6 | 0 | 1 | 100.0% | 85.7% |
|
||||
| status | 7 | 6 | 1 | 1 | 85.7% | 85.7% |
|
||||
| tpm | 5 | 4 | 0 | 1 | 100.0% | 80.0% |
|
||||
| region | 2 | 1 | 0 | 1 | 100.0% | 50.0% |
|
||||
|
||||
## source_silent-diagnostikk
|
||||
|
||||
Judgen hentet siden men fant ikke verdien. Diagnostisk, ikke et flagg.
|
||||
|
||||
| signal | antall | tolkning |
|
||||
|---|---|---|
|
||||
| På verifiserbar feil | 2 | judge-bom: reell feil oversett via «kan ikke verifisere» |
|
||||
| På verifiserbar correct | 3 | judge reproduserte ikke et korrekt faktum mennesket fant |
|
||||
| Enig med unsourced | 5 | judge reproduserer den uverifiserbare grensen (godt) |
|
||||
| Uenig med unsourced | 10 | judge hevdet grunnet/ugrunnet der mennesket ikke fant kilde |
|
||||
|
||||
## GATE: ✅ PASS — bygg S3
|
||||
|
||||
- recall 0.842 ≥ 0.8? **ja**
|
||||
- presisjon 0.842 ≥ 0.7? **ja**
|
||||
- slår staleness (recall 0.000)? **ja**
|
||||
- begrunnelse: all criteria met
|
||||
1030
scripts/kb-eval/data/judge-bakeoff-results-v2.json
Normal file
1030
scripts/kb-eval/data/judge-bakeoff-results-v2.json
Normal file
File diff suppressed because it is too large
Load diff
110
scripts/kb-eval/judge-claim-prompt-v2.md
Normal file
110
scripts/kb-eval/judge-claim-prompt-v2.md
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
# Per-claim groundedness judge — S1 bake-off **v2** (targeted iteration)
|
||||
|
||||
v2 of `judge-claim-prompt.md`. Same blind, per-claim, one-subagent-per-file design.
|
||||
**Why v2 exists (transparent, not p-hacking):** v1 FAILED the pre-registered gate
|
||||
(recall 68.4%, frozen as the honest result). The misses were concentrated and
|
||||
diagnosable — 10 of 12 false negatives were `grounded`-but-wrong: the judge fetched
|
||||
a page, found a quote it read as supporting the claim, but the asserted value had
|
||||
actually drifted (worst on `sku`: recall 37.5%). v2 fixes exactly that reasoning
|
||||
error with a **stricter exact-value entailment rule** — a general correctness
|
||||
improvement to the judge's standard, defensible independent of the test outcome. v2
|
||||
does NOT touch the thresholds and does NOT loosen any precision criterion.
|
||||
|
||||
The v1 result stays frozen (`judge-bakeoff-results.json`, `...-report.*`). v2 writes
|
||||
to `judge-bakeoff-results-v2.json` and is graded against the same frozen gold set.
|
||||
|
||||
---
|
||||
|
||||
You are a correctness judge for Microsoft AI reference documentation. You verify
|
||||
factual claims against **live, official Microsoft Learn** (`learn.microsoft.com`).
|
||||
Be strict and adversarial — do not give the benefit of the doubt, do not pad, do not
|
||||
infer a value the source does not state.
|
||||
|
||||
You are judging claims extracted from `<FILE>`. For EACH claim in the batch below,
|
||||
decide whether the cited Microsoft Learn source **grounds** the claim.
|
||||
|
||||
## The three verdicts (exhaustive, mutually exclusive)
|
||||
|
||||
- **`grounded`** — you fetched a `learn.microsoft.com` page that states the claimed
|
||||
value(s). The page supports the claim. (Maps to gold `correct`.)
|
||||
- **`not_grounded`** — you fetched a `learn.microsoft.com` page that states a
|
||||
**different / contradicting / superseded** value for what the claim asserts. The
|
||||
claim disagrees with the source. (Maps to gold `outdated` + `wrong`.)
|
||||
- **`source_silent`** — you fetched the cited page (and searched as a fallback) but
|
||||
**no** `learn.microsoft.com` page states the claimed value at all. You cannot
|
||||
confirm or refute it. (Maps to gold `unsourced`.) Pricing on JS-rendered Azure
|
||||
pages typically lands here — that is expected, not a failure.
|
||||
|
||||
## ⚠️ EXACT-VALUE RULE (the v2 sharpening — read carefully)
|
||||
|
||||
The most common v1 error was calling a claim `grounded` because the page **discussed
|
||||
the same topic/SKU/model**, while the specific asserted value had actually drifted.
|
||||
Fix that:
|
||||
|
||||
- A claim is `grounded` ONLY if the fetched page states the **exact** asserted
|
||||
value(s). Verifying that the page "is about" the SKU/model/feature is **not**
|
||||
enough — the specific number, name, date, tier, dimension, or status must match.
|
||||
- If the claim asserts value **X** and the page states a **different** value **Y**
|
||||
(even if Y is adjacent, plausible, or a near-miss), the verdict is **`not_grounded`**,
|
||||
not `grounded`. Do not round, approximate, or accept "close enough."
|
||||
- This applies with special force to:
|
||||
- **`sku`** — exact SKU/tier name, exact PTU minimum/increment, exact deployment
|
||||
type. A different SKU value on the page = `not_grounded`.
|
||||
- **`taxonomy`** — the exact categorization/mapping. If the page maps the item
|
||||
differently (different category, different which-does-what), that is `not_grounded`.
|
||||
- **`version` / `tpm` / `region` / `status`** — exact date/number/region/GA-preview
|
||||
status. A superseded date or changed number is `not_grounded`.
|
||||
- This rule does NOT lower the bar for `not_grounded`: you still need a fetched
|
||||
`learn.microsoft.com` quote that states the **differing** value. "I couldn't find
|
||||
the value" remains `source_silent`, never `not_grounded`.
|
||||
|
||||
So: be **stricter about what counts as `grounded`** (exact match required), while
|
||||
keeping the same evidence discipline for `not_grounded` and `source_silent`.
|
||||
|
||||
A claim is `not_grounded` if the source contradicts **any** checkable value in it.
|
||||
It is `grounded` only if the source supports **all** checkable values exactly. If the
|
||||
source states none of them, it is `source_silent`.
|
||||
|
||||
## Procedure (per claim)
|
||||
|
||||
1. **Identify the volatile assertion(s)** in the claim text. The `claim_type` tells
|
||||
you what to check:
|
||||
- `version` → model/API version, GA date, context window, max output, training cutoff
|
||||
- `tpm` → tokens-per-minute / throughput / quota numbers
|
||||
- `sku` → SKU name, tier, PTU minimums, deployment type
|
||||
- `region` → regional availability
|
||||
- `status` → GA / preview / retirement / deprecation status
|
||||
- `taxonomy` → categorization, capability mapping, which-feature-does-what
|
||||
2. **Fetch the cited source** with `microsoft_docs_fetch` on the claim's
|
||||
`evidence_url`. If the claim has no `evidence_url`, or the fetched page does not
|
||||
address the assertion, run `microsoft_docs_search` to find the authoritative page.
|
||||
3. **Exact-value entailment check** each checkable value against the fetched text
|
||||
(apply the EXACT-VALUE RULE above).
|
||||
4. **Strict evidence rule:** a `grounded` or `not_grounded` verdict REQUIRES a
|
||||
verbatim quote you actually fetched from a `learn.microsoft.com` URL that states
|
||||
the relevant value. No quote → `source_silent`. Never quote from memory.
|
||||
|
||||
## Hard rules
|
||||
|
||||
- Verify against the fetched page only. Do not rely on prior knowledge of model
|
||||
specs / prices — those are exactly what may have drifted.
|
||||
- Stable identifiers are not volatile and are not your job to refute: regulation year
|
||||
(2024/1689), case numbers (C-311/18), standard version names (OWASP LLM Top 10
|
||||
2025, MADR v3.0), file names. If a claim is purely such an identifier, judge it on
|
||||
whatever volatile value it carries, else `source_silent`.
|
||||
- One verdict per claim. Return EXACTLY the JSON below — no prose, no markdown fence.
|
||||
- `evidence_quote` = the verbatim sentence/value from the fetched page that drove the
|
||||
verdict (empty string for `source_silent`). `evidence_url` = the page you actually
|
||||
used (may differ from the cited one if you fell back to search).
|
||||
|
||||
## Batch to judge (from `<FILE>`)
|
||||
|
||||
<CLAIMS>
|
||||
|
||||
## Output (strict JSON, no fence)
|
||||
|
||||
```
|
||||
{"file":"<FILE>","results":[
|
||||
{"id":"<claim id>","judge_verdict":"grounded|not_grounded|source_silent","evidence_url":"<url actually used>","evidence_quote":"<verbatim quote or empty>","reason":"<one sentence: what the source said vs the claim>"}
|
||||
]}
|
||||
```
|
||||
|
|
@ -36,7 +36,10 @@ if (!Number.isFinite(minRecall) || !Number.isFinite(minPrecision)) {
|
|||
const thresholds = { minRecall, minPrecision };
|
||||
|
||||
const gold = JSON.parse(fs.readFileSync(path.join(DATA, 'gold-correctness-set.json'), 'utf8'));
|
||||
const resultsPath = path.join(DATA, 'judge-bakeoff-results.json');
|
||||
// --results / --report-prefix let a second iteration (v2) be graded without
|
||||
// clobbering the frozen v1 artifacts. Defaults preserve the v1 file names.
|
||||
const resultsPath = path.join(DATA, flag('--results') || 'judge-bakeoff-results.json');
|
||||
const reportPrefix = flag('--report-prefix') || 'judge-bakeoff-report';
|
||||
if (!fs.existsSync(resultsPath)) {
|
||||
console.error(`error: ${resultsPath} not found — run the judge fan-out first`);
|
||||
process.exit(2);
|
||||
|
|
@ -126,8 +129,8 @@ Judgen hentet siden men fant ikke verdien. Diagnostisk, ikke et flagg.
|
|||
`;
|
||||
|
||||
if (process.argv.includes('--write')) {
|
||||
const jsonOut = path.join(DATA, 'judge-bakeoff-report.json');
|
||||
const mdOut = path.join(DATA, 'judge-bakeoff-report.md');
|
||||
const jsonOut = path.join(DATA, `${reportPrefix}.json`);
|
||||
const mdOut = path.join(DATA, `${reportPrefix}.md`);
|
||||
fs.writeFileSync(
|
||||
jsonOut,
|
||||
JSON.stringify(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue