From a0283efe84d27a839e6adc895ff9793c2e8d4713 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Sat, 4 Jul 2026 23:24:06 +0200 Subject: [PATCH] docs(ms-ai-architect): R11-flaggformat-spec + valideringstest (judge-output + augmentering) [skip-docs] --- docs/r11-flag-format-2026-07.md | 92 +++++++++++++++++++ .../kb-eval/test-judge-pass-manifest.test.mjs | 59 ++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 docs/r11-flag-format-2026-07.md diff --git a/docs/r11-flag-format-2026-07.md b/docs/r11-flag-format-2026-07.md new file mode 100644 index 0000000..2bd344a --- /dev/null +++ b/docs/r11-flag-format-2026-07.md @@ -0,0 +1,92 @@ +# R11 flag format — the judge-pass → fix-work-list contract + +**Spec for the flag record the R7–R10 judge pass emits and R11 (the fix step) +consumes. A flag is a judged claim the pass could NOT ground against its cited +Microsoft Learn source. The pass never fixes — it stamps (born-verified) or +flags; R11 does the human-confirmed fix.** + +Status: design spec (R6 Step 5). Builds on the frozen v3.1 claim judge +(`scripts/kb-eval/judge-claim-prompt-v3.1.md`) and the judge-pass ledger +(`scripts/kb-eval/judge-pass-manifest.mjs`, R6 Step 4). Consumed by R11. + +--- + +## Where a flag comes from + +The v3.1 judge returns, per claim, a strict-JSON result (no fence): + +``` +{"file":"","results":[ + {"id":"","judge_verdict":"grounded|not_grounded|source_silent", + "rule":"","evidence_url":"", + "evidence_quote":"","reason":""} +]} +``` + +The pass augments each **non-grounded** result (`judge_verdict !== 'grounded'`) +into a flag record. This is a **minimal augmentation, not a pass-through** — the +judge output is a subset of the flag record; the pass adds four fields from the +fan-out context and the claim manifest: + +| Added field | Source | +|-------------|--------| +| `file` | the fan-out context (one subagent per file) — also echoed by the judge output's top-level `file` | +| `line` | the claim manifest (`scripts/kb-eval/extract-judge-claims.mjs` output) | +| `claim` | the claim manifest (the verbatim claim text that was judged) | +| `disposition` | mapped from `judge_verdict` (see the vocabulary mapping below) | + +## The flag record + +A flag record is the union of the judge result and the four augmented fields: + +``` +{ + "id": "", + "judge_verdict": "not_grounded" | "source_silent", // 'grounded' is never flagged + "rule": "R1".."R8" | "", + "evidence_url": "", + "evidence_quote":"", + "reason": "", + "file": "skills/.../x.md", + "line": , + "claim": "", + "disposition": "outdated" | "wrong" | "unsourced" // R11 fix disposition +} +``` + +In the ledger (R6 Step 4), a flagged file is a manifest record with +`per_file_verdict: "flagged"` and one `flags[]` entry per non-grounded claim — +the structural mirror of `spor0-fix-manifest.json`'s "fix-record with +`applied:false`". A file is `flagged` if **any** judgeable claim is non-grounded +(the born-verified rule is AND: `pass` requires EVERY claim grounded). + +## The two vocabularies + the mapping + +There are two distinct enumerations, and conflating them is the trap this spec +exists to prevent: + +1. **`judge_verdict`** — the judge's per-claim output (v3.1 prompt): + - `grounded` — the source confirms the claim → never flagged. + - `not_grounded` — the source contradicts / does not support the claim. + - `source_silent` — the source does not address the claim at all. + +2. **`disposition`** — the R11 fix disposition, the correctness vocabulary from + `scripts/kb-eval/lib/base-rate.mjs` (`VERDICTS`): `correct`, `outdated`, + `wrong`, `unsourced`. Its `ERROR_VERDICTS` subset — `outdated` and `wrong` — + are the only ones R11 treats as **fix targets**. `unsourced` is recorded but + is not, on its own, a content error (no MS source confirms it either way). + +**Mapping** (`judge_verdict` → `disposition`): + +| `judge_verdict` | `disposition` | R11 fix target? | +|-----------------|---------------|-----------------| +| `grounded` | `correct` | no (never flagged) | +| `not_grounded` | `outdated` or `wrong` | **yes** — the human assigns which at R11 | +| `source_silent` | `unsourced` | no (flagged, but not a fix target) | + +`not_grounded → {outdated, wrong}` is deliberately a one-to-two mapping: the +judge establishes the claim is not grounded; whether it is stale-but-once-true +(`outdated`) or never-true (`wrong`) is a human call at R11, re-verified against +the live source (verification duty). The pass never auto-fixes — every flag is +human-confirmed in R11, so a judge false-positive becomes a human review, never +silent corruption of a public file. diff --git a/tests/kb-eval/test-judge-pass-manifest.test.mjs b/tests/kb-eval/test-judge-pass-manifest.test.mjs index b268de9..0f7cdbb 100644 --- a/tests/kb-eval/test-judge-pass-manifest.test.mjs +++ b/tests/kb-eval/test-judge-pass-manifest.test.mjs @@ -170,3 +170,62 @@ test('CLI unknown flag → usage error exit 2', () => { } assert.equal(status, 2); }); + +// --- R11 flag format (R6 Step 5): the judge-pass → fix-work-list contract ----------- +// A flag = a judged claim the pass could not ground. The pass augments the judge output +// {id, judge_verdict, rule, evidence_url, evidence_quote, reason} with {file, line, claim, +// disposition}. Two vocabularies must never be conflated: judge_verdict (grounded/ +// not_grounded/source_silent) vs disposition (base-rate VERDICTS; ERROR_VERDICTS = R11 fix +// targets). Spec: docs/r11-flag-format-2026-07.md. + +const JUDGE_VERDICTS = ['grounded', 'not_grounded', 'source_silent']; +const DISPOSITIONS = ['correct', 'outdated', 'wrong', 'unsourced']; // mirrors base-rate VERDICTS +const ERROR_DISPOSITIONS = ['outdated', 'wrong']; // mirrors base-rate ERROR_VERDICTS (R11 fix targets) + +const FLAG = { + id: 'azure-ai-foundry.md#3', judge_verdict: 'not_grounded', rule: 'R3', + evidence_url: 'https://learn.microsoft.com/azure/ai-foundry/models', evidence_quote: 'GPT-4o mini context 128k', + reason: 'File claims X tokens; source states Y.', + file: 'skills/ms-ai-advisor/references/platforms/azure-ai-foundry.md', line: 42, + claim: 'DeepSeek-R1 131072 tokens', disposition: 'outdated', +}; + +test('flag record carries the judge output fields + the four augmented fields', () => { + for (const k of ['id', 'judge_verdict', 'rule', 'evidence_url', 'evidence_quote', 'reason', 'file', 'line', 'claim', 'disposition']) { + assert.ok(k in FLAG, `flag record missing '${k}'`); + } + assert.ok(JUDGE_VERDICTS.includes(FLAG.judge_verdict) && FLAG.judge_verdict !== 'grounded', 'a flag is never grounded'); + assert.match(FLAG.rule, /^(R[1-8])?$/); // R1-R8 or empty + assert.ok(DISPOSITIONS.includes(FLAG.disposition)); +}); + +test('a flagged ledger record built from a flag validates (Step 4 ↔ Step 5 contract)', () => { + const rec = { + file: FLAG.file, batch: 1, judged_at: '2026-07-04', per_file_verdict: 'flagged', + claim_count: 8, verified: null, verified_by: null, flags: [FLAG], + }; + assert.equal(validateManifest(appendJudgedFile(scaffoldManifest(), rec)).valid, true); +}); + +test('ERROR dispositions (outdated/wrong) are the R11 fix targets; unsourced/correct are not', () => { + assert.deepEqual(ERROR_DISPOSITIONS, ['outdated', 'wrong']); + assert.ok(!ERROR_DISPOSITIONS.includes('unsourced')); + assert.ok(!ERROR_DISPOSITIONS.includes('correct')); +}); + +test('disposition vocabulary is parity-checked against base-rate.mjs source (no silent drift)', () => { + // base-rate.mjs is the source of truth for the correctness vocabulary; the flag disposition + // reuses it (its enum is module-private, so we assert against the source rather than duplicate + // a live copy). If base-rate's vocabulary changes, this fails → drift is caught. + const src = readFileSync(join(__dirname, '..', '..', 'scripts', 'kb-eval', 'lib', 'base-rate.mjs'), 'utf8'); + for (const v of DISPOSITIONS) assert.match(src, new RegExp(`'${v}'`), `base-rate must define '${v}'`); + assert.match(src, /ERROR_VERDICTS\s*=\s*new Set\(\[\s*'outdated'\s*,\s*'wrong'\s*\]\)/); +}); + +test('the R11 flag-format spec documents judge_verdict + the disposition mapping', () => { + const doc = readFileSync(join(__dirname, '..', '..', 'docs', 'r11-flag-format-2026-07.md'), 'utf8'); + assert.match(doc, /judge_verdict/); + assert.match(doc, /not_grounded/); + assert.match(doc, /disposition/); + for (const v of DISPOSITIONS) assert.ok(doc.includes(v), `doc must document disposition '${v}'`); +});