fix(ms-ai-architect): R11 — første korpus-edit; 4 ratifiserte O2-subtraksjoner anvendt (idx 17, 19, 33, 14-redusert)
Operatør-ratifisert 2026-08-03. Sletter udokumenterte påstander fra tre KB-filer: score-threshold-båndene og 'Automatic indexing av vectors' (rag-caching-optimization), to ikke-støttede AISPM-attribusjoner (ai-threat-modeling-stride), og SharePoint som feedback-lagring (feedback-loops). idx 14 er den REDUSERTE subtraksjonen — 'Automatically' beholdes, siden linje 566 hevder automatikk. Ikke ratifisert og ikke anvendt: idx 26, 27, 36, 18. Driver ankrer på file_text_verbatim, aldri linjenummer, og avbryter uten å skrive ved tvetydig anker, ikke-sletting eller ny ordform. 11 tester. Suite 1032/1032. [skip-docs]
This commit is contained in:
parent
4042d0b94a
commit
957ebef6da
5 changed files with 231 additions and 7 deletions
149
scripts/kb-eval/apply-o2-ratified.mjs
Normal file
149
scripts/kb-eval/apply-o2-ratified.mjs
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
#!/usr/bin/env node
|
||||
// apply-o2-ratified.mjs — R11 §9.4. Applies the O2 subtractions the operator
|
||||
// ratified 2026-08-03, and only those.
|
||||
//
|
||||
// Ratified: idx 17, 19, 33 as attested; idx 14 as the REDUCED subtraction
|
||||
// (`/ SharePoint` only). Deliberately NOT ratified and therefore absent from
|
||||
// RATIFIED: idx 26 (renumbering artifact unresolved), idx 27 (cond 3 still
|
||||
// `human_must_confirm`), idx 36 (needs the out-of-envelope companion edit at
|
||||
// line 310, now owned by gap G7), idx 18 (no reduction exists — also G7).
|
||||
//
|
||||
// Every string comes from the tracked evidence in data/r11-o2-returns/, never
|
||||
// from transcription. The one amendment (idx 14) is expressed as a derivation
|
||||
// over the attested verbatim and asserts its own effect, so a drifted record
|
||||
// aborts rather than silently writing something else.
|
||||
//
|
||||
// Anchoring is on `file_text_verbatim`, NEVER on a line number: `line` differs
|
||||
// from `real_line` in 9 of 17 records, and idx 17 shifts idx 19's lines in the
|
||||
// file they share. The verbatim must occur EXACTLY once or the run aborts.
|
||||
//
|
||||
// Recovery contract: writes are crash-safe (atomicWriteSync tmp+rename — a reader
|
||||
// sees the old file or the new one, never a partial). An interrupted run is
|
||||
// recovered by re-running: an already-applied edit no longer finds its verbatim,
|
||||
// which aborts the run, writing nothing, rather than corrupting the file.
|
||||
//
|
||||
// Usage: node scripts/kb-eval/apply-o2-ratified.mjs [--dry]
|
||||
import { readFileSync, readdirSync, realpathSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { isDeletionOnly, novelWordForms } from './lib/o2-return-check.mjs';
|
||||
import { atomicWriteSync } from '../kb-update/lib/atomic-write.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PLUGIN_ROOT = join(__dirname, '..', '..');
|
||||
const RETURNS = join(PLUGIN_ROOT, 'scripts/kb-eval/data/r11-o2-returns');
|
||||
|
||||
/**
|
||||
* The ratified reduction for idx 14: delete only the `/ SharePoint` alternative.
|
||||
* `Automatically` MUST survive — line 566 of the same file restates automaticity
|
||||
* in Norwegian, so deleting it would leave a remainder the file contradicts.
|
||||
* @param {string} verbatim
|
||||
* @returns {string}
|
||||
*/
|
||||
export function reduceSharePointOnly(verbatim) {
|
||||
const out = verbatim.replace('Dataverse / SharePoint', 'Dataverse');
|
||||
if (out === verbatim) throw new Error('idx 14 reduction is a no-op — record drifted');
|
||||
if (!out.includes('Automatically add')) throw new Error('idx 14: `Automatically` must survive');
|
||||
return out;
|
||||
}
|
||||
|
||||
// Frozen manifest — the operator's ratification, 2026-08-03. `amend: null` means
|
||||
// apply the attested `proposed_remainder` byte-for-byte.
|
||||
export const RATIFIED = [
|
||||
{ idx: 17, amend: null },
|
||||
{ idx: 19, amend: null },
|
||||
{ idx: 33, amend: null },
|
||||
{ idx: 14, amend: reduceSharePointOnly },
|
||||
];
|
||||
|
||||
/**
|
||||
* The remainder actually written for a record: attested, or the ratified amendment.
|
||||
* @param {object} row
|
||||
* @param {{amend: ((v: string) => string) | null}} entry
|
||||
* @returns {string}
|
||||
*/
|
||||
export function amendedRemainder(row, entry) {
|
||||
return entry.amend ? entry.amend(row.file_text_verbatim) : row.proposed_remainder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the anchored block with its remainder. Pure — no I/O. Throws on any
|
||||
* condition that would make the write unsafe rather than writing something else.
|
||||
* @param {string} content
|
||||
* @param {string} verbatim
|
||||
* @param {string} remainder
|
||||
* @returns {string}
|
||||
*/
|
||||
export function applyEdit(content, verbatim, remainder) {
|
||||
const occurrences = content.split(verbatim).length - 1;
|
||||
if (occurrences !== 1) {
|
||||
throw new Error(`ABORT — anchor occurs ${occurrences} times, expected exactly 1`);
|
||||
}
|
||||
if (!isDeletionOnly(verbatim, remainder)) {
|
||||
throw new Error('ABORT — remainder is not deletion-only w.r.t. the anchor');
|
||||
}
|
||||
const novel = novelWordForms(verbatim, remainder);
|
||||
if (novel.length) {
|
||||
throw new Error(`ABORT — remainder introduces novel word form(s): ${novel.join(', ')}`);
|
||||
}
|
||||
return content.replace(verbatim, remainder);
|
||||
}
|
||||
|
||||
function loadRows() {
|
||||
return readdirSync(RETURNS).filter((f) => f.endsWith('.json')).sort()
|
||||
.flatMap((f) => JSON.parse(readFileSync(join(RETURNS, f), 'utf8')));
|
||||
}
|
||||
|
||||
export function run({ dry = false } = {}) {
|
||||
const byIdx = new Map(loadRows().map((r) => [r.idx, r]));
|
||||
|
||||
// Group by file so two edits sharing a file compose in memory and write once.
|
||||
const perFile = new Map();
|
||||
for (const entry of RATIFIED) {
|
||||
const row = byIdx.get(entry.idx);
|
||||
if (!row) throw new Error(`ABORT — no return record for idx ${entry.idx}`);
|
||||
if (row.verdict !== 'O2_CANDIDATE') {
|
||||
throw new Error(`ABORT — idx ${entry.idx} is ${row.verdict}, not an O2 candidate`);
|
||||
}
|
||||
if (!perFile.has(row.file)) perFile.set(row.file, []);
|
||||
perFile.get(row.file).push({ entry, row });
|
||||
}
|
||||
|
||||
const planned = [];
|
||||
for (const [rel, edits] of perFile) {
|
||||
const abs = join(PLUGIN_ROOT, rel);
|
||||
const before = readFileSync(abs, 'utf8');
|
||||
let out = before;
|
||||
for (const { entry, row } of edits) {
|
||||
out = applyEdit(out, row.file_text_verbatim, amendedRemainder(row, entry));
|
||||
}
|
||||
// Post-condition: every anchor is gone, and the file actually changed.
|
||||
for (const { row } of edits) {
|
||||
if (out.includes(row.file_text_verbatim)) {
|
||||
throw new Error(`ABORT — idx ${row.idx} anchor still present after edit`);
|
||||
}
|
||||
}
|
||||
if (out === before) throw new Error(`ABORT — ${rel} unchanged`);
|
||||
planned.push({ rel, abs, out, idxs: edits.map((e) => e.entry.idx) });
|
||||
}
|
||||
|
||||
console.log(`Ratified edits: ${RATIFIED.length} across ${planned.length} files`);
|
||||
for (const p of planned) console.log(` ~ ${p.rel} (idx ${p.idxs.join(', ')})`);
|
||||
if (dry) {
|
||||
console.log('\n(dry run — no writes)');
|
||||
return planned;
|
||||
}
|
||||
for (const p of planned) atomicWriteSync(p.abs, p.out);
|
||||
console.log(`\nWrote ${planned.length} files.`);
|
||||
return planned;
|
||||
}
|
||||
|
||||
const isMain = (() => {
|
||||
try {
|
||||
return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})();
|
||||
if (isMain) run({ dry: process.argv.includes('--dry') });
|
||||
|
|
@ -552,7 +552,7 @@ ml_client.schedules.begin_create_or_update(model_monitor)
|
|||
| Komponent | Power Platform-tjeneste | Formål |
|
||||
|-----------|-------------------------|--------|
|
||||
| **Automated feedback collection** | Power Automate | Route low-confidence predictions til human review |
|
||||
| **Storage** | Dataverse / SharePoint | Lagre feedback data |
|
||||
| **Storage** | Dataverse | Lagre feedback data |
|
||||
| **Model improvement** | AI Builder Feedback Loop | Automatically add reviewed samples to training set |
|
||||
| **Retraining** | AI Builder | Manual/scheduled retraining |
|
||||
|
||||
|
|
|
|||
|
|
@ -251,9 +251,6 @@ Outbound (cache store):
|
|||
```
|
||||
|
||||
**Score Threshold Tuning** (APIM `score-threshold` er en DISTANSE: lavere = strengere, krever høyere semantisk likhet):
|
||||
- 0.1-0.2 → Strict matching, lavere hit rate, høy relevance
|
||||
- 0.3-0.5 → Balanced, medium hit rate, god relevance
|
||||
- 0.6-0.8 → Liberal matching, høyere hit rate, noe lavere relevance
|
||||
|
||||
**Verified** (Microsoft Learn - Enable semantic caching for LLM APIs)
|
||||
|
||||
|
|
@ -294,7 +291,6 @@ def query_cache(prompt_vector, similarity_threshold=0.15, top_k=5):
|
|||
|
||||
**Fordeler:**
|
||||
- Globally distributed, multi-region writes
|
||||
- Automatic indexing av vectors
|
||||
- 99.999% SLA med multi-region setup
|
||||
- Built-in TTL support
|
||||
|
||||
|
|
|
|||
|
|
@ -208,10 +208,9 @@ STRIDE Mapping: Tampering
|
|||
### Microsoft Defender for Cloud — AI Security Posture Management
|
||||
|
||||
**Capabilities:** *(Verified MCP 2026-04)*
|
||||
- Automated detection of AI workloads across Azure subscriptions (via Azure Resource Graph)
|
||||
- Automated detection of AI workloads across Azure subscriptions
|
||||
- AI security posture management: automate detection and remediation of generative AI risks
|
||||
- Security recommendations for AI models, data stores, network isolation
|
||||
- Integration with Purview for data classification, DLP og Insider Risk Management for prompt-based data exfiltration
|
||||
|
||||
**Threat Modeling Integration:**
|
||||
```plaintext
|
||||
|
|
|
|||
80
tests/kb-eval/test-apply-o2-ratified.test.mjs
Normal file
80
tests/kb-eval/test-apply-o2-ratified.test.mjs
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
// test-apply-o2-ratified.test.mjs — R11 §9.4, the apply half.
|
||||
//
|
||||
// The driver edits PUBLICLY DISTRIBUTED KB files, so every invariant that stands
|
||||
// between a ratified string and a write is tested here: exactly-once anchoring,
|
||||
// deletion-only, composition of two edits in one file, and idempotency.
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { applyEdit, amendedRemainder, reduceSharePointOnly, RATIFIED }
|
||||
from '../../scripts/kb-eval/apply-o2-ratified.mjs';
|
||||
|
||||
const FILE = ['# Title', '', 'alpha', 'BLOCK line one', 'BLOCK line two', '', 'omega', ''].join('\n');
|
||||
|
||||
test('applyEdit replaces the anchored block and leaves the rest byte-identical', () => {
|
||||
const out = applyEdit(FILE, 'BLOCK line one\nBLOCK line two', 'BLOCK line one');
|
||||
assert.equal(out, ['# Title', '', 'alpha', 'BLOCK line one', '', 'omega', ''].join('\n'));
|
||||
});
|
||||
|
||||
test('applyEdit throws when the verbatim does not occur', () => {
|
||||
assert.throws(() => applyEdit(FILE, 'ABSENT', 'ABSENT'), /occurs 0 times/);
|
||||
});
|
||||
|
||||
test('applyEdit throws when the verbatim occurs more than once (ambiguous anchor)', () => {
|
||||
const dup = FILE + 'BLOCK line one\nBLOCK line two\n';
|
||||
assert.throws(() => applyEdit(dup, 'BLOCK line one\nBLOCK line two', 'BLOCK line one'), /occurs 2 times/);
|
||||
});
|
||||
|
||||
test('applyEdit refuses a remainder that is not deletion-only', () => {
|
||||
assert.throws(
|
||||
() => applyEdit(FILE, 'BLOCK line one\nBLOCK line two', 'BLOCK line one\nBLOCK line THREE'),
|
||||
/not deletion-only/,
|
||||
);
|
||||
});
|
||||
|
||||
test('applyEdit refuses a remainder that introduces a novel word form (V2b)', () => {
|
||||
// 'Automatically add' -> 'Add' passes a character subsequence but recapitalises.
|
||||
assert.throws(
|
||||
() => applyEdit('x\nAutomatically add samples\ny', 'Automatically add samples', 'Add samples'),
|
||||
/novel word form/,
|
||||
);
|
||||
});
|
||||
|
||||
test('two edits in the same file compose, and the second is unaffected by the first shifting lines', () => {
|
||||
const src = ['head', 'AAA one', 'AAA two', 'mid', 'BBB one', 'BBB two', 'tail'].join('\n');
|
||||
let out = applyEdit(src, 'AAA one\nAAA two', 'AAA one');
|
||||
out = applyEdit(out, 'BBB one\nBBB two', 'BBB one');
|
||||
assert.equal(out, ['head', 'AAA one', 'mid', 'BBB one', 'tail'].join('\n'));
|
||||
});
|
||||
|
||||
test('applyEdit is idempotent-safe: re-running on an applied file throws rather than corrupting', () => {
|
||||
const once = applyEdit(FILE, 'BLOCK line one\nBLOCK line two', 'BLOCK line one');
|
||||
assert.throws(() => applyEdit(once, 'BLOCK line one\nBLOCK line two', 'BLOCK line one'), /occurs 0 times/);
|
||||
});
|
||||
|
||||
test('reduceSharePointOnly drops the SharePoint alternative and KEEPS Automatically', () => {
|
||||
const verbatim = [
|
||||
'| **Storage** | Dataverse / SharePoint | Lagre feedback data |',
|
||||
'| **Model improvement** | AI Builder Feedback Loop | Automatically add reviewed samples to training set |',
|
||||
].join('\n');
|
||||
const out = reduceSharePointOnly(verbatim);
|
||||
assert.match(out, /\| Dataverse \|/);
|
||||
assert.doesNotMatch(out, /SharePoint/);
|
||||
assert.match(out, /Automatically add reviewed samples/, 'line 566 asserts automaticity — it must survive');
|
||||
});
|
||||
|
||||
test('reduceSharePointOnly throws if the surgery is a no-op (guards against silent drift)', () => {
|
||||
assert.throws(() => reduceSharePointOnly('nothing to reduce'), /no-op/);
|
||||
});
|
||||
|
||||
test('RATIFIED carries exactly the four operator-ratified edits, and not the three held back', () => {
|
||||
assert.deepEqual(RATIFIED.map((r) => r.idx).sort((a, b) => a - b), [14, 17, 19, 33]);
|
||||
for (const held of [26, 27, 36]) {
|
||||
assert.equal(RATIFIED.some((r) => r.idx === held), false, `idx ${held} was NOT ratified`);
|
||||
}
|
||||
});
|
||||
|
||||
test('amendedRemainder returns the attested string untouched when there is no amendment', () => {
|
||||
const row = { idx: 17, proposed_remainder: 'kept', file_text_verbatim: 'kept\ndropped' };
|
||||
assert.equal(amendedRemainder(row, { idx: 17, amend: null }), 'kept');
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue