chore(ms-ai-architect): fasett-bake-off mot STRUCTURED_V1, maalt paa eget korpus
Ren maaling, ingen produksjonskodeendring: eval-driveren importerer scripts/okf-consume/, og ingen runtime importerer den. Fire fasettprofiler + to referansearmer, nevner 40 gullspoersmaal. Tre-noekkel-settet (status, supersedes, derived) holder treffet HELT oppe mot full STRUCTURED_V1 - identisk celle for celle - til 1,59x flat indeks mot 1,65x. STRUCTURED_V1s fem oevrige noekler kjoeper null. Dekomposisjon: tittel-labelen alene kjoeper hele gjenfinningsgevinsten (6/40 -> 11/40), status kjoeper hele felle-gevinsten (5/20 -> 0/20). Mot hodeskann: 11/40 og 0/20 til 8909 tegn mot 12/40 og 0/20 til 41416. Tre negative/kjent-positive kontroller vedlagt i artefaktkatalogen.
This commit is contained in:
parent
c0f10618da
commit
24cb3f14f0
1 changed files with 224 additions and 0 deletions
224
scripts/kb-eval/measure-okf-facets.mjs
Normal file
224
scripts/kb-eval/measure-okf-facets.mjs
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
#!/usr/bin/env node
|
||||
// measure-okf-facets.mjs — EVAL DRIVER for the STRUCTURED_V1 facet bake-off.
|
||||
//
|
||||
// Answers order 20260827T084938Z-4734427711: is a FACETED OKF index worth its
|
||||
// character cost on THIS corpus, against THESE 40 gold questions?
|
||||
//
|
||||
// This is eval code, exactly like measure-okf-consume.mjs. No agent reads it and
|
||||
// nothing in the plugin runtime imports it; it imports scripts/okf-consume/,
|
||||
// never the other way round. No production behaviour changes with this file.
|
||||
//
|
||||
// Run: node scripts/kb-eval/measure-okf-facets.mjs [--json]
|
||||
// Needs ~/okf-bakeoff-2026-08/ with the bundles built by bygg/build-bundles.py.
|
||||
|
||||
import { readFileSync, readdirSync, existsSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { homedir } from 'node:os';
|
||||
import {
|
||||
tokenize, tokensMatch, headScan, scoreCandidate, cutToBudget,
|
||||
AGENT_BUDGET, HEAD_SCAN_LINES,
|
||||
} from '../okf-consume/lib/bundle-cut.mjs';
|
||||
import { buildOrgSummary, ORG_FILES, FREE_CONTEXT_FILE } from '../kb-update/lib/user-data.mjs';
|
||||
|
||||
const S = join(homedir(), 'okf-bakeoff-2026-08');
|
||||
const ORG = join(homedir(), '.claude', 'ms-ai-architect', 'org');
|
||||
const MAPPER = ['arkitektur', 'hendelser', 'leverandor', 'styring'];
|
||||
|
||||
// Instrument gate. The two CORPUS-side anchors from the 2026-08-27 consume
|
||||
// measurement, which are what this measurement actually rests on. The bundle-side
|
||||
// anchor (heleBundlen 45580) is NOT here: this session rebuilt the control bundle
|
||||
// with llm-ingestion-okf @ d35bcb2 and lost the 01e4170 artefact, so the old value
|
||||
// is unreproducible. The new value is re-pinned below and the loss is reported.
|
||||
const KJENT_GODE = { a1: 31343, a2a: 2884 };
|
||||
const REPINNET = { heleBundlen: 45898, indeksFlat: 4521, kilde: 'llm-ingestion-okf @ d35bcb2' };
|
||||
|
||||
// The index line grammar STRUCTURED_V1 renders: DEFAULT's link plus a facet tail.
|
||||
const INDEKS_LINJE = /^- \[([^\]]*)\]\(([^)]+)\)(?: — (.*))?$/;
|
||||
const FASETT = /^([a-zA-Z_æøåÆØÅ]+):\s*(.+)$/;
|
||||
const ID_FRA_STAMME = /-([a-z]\d{2})-/;
|
||||
|
||||
function lesKorpus() {
|
||||
const ut = [];
|
||||
for (const mappe of MAPPER) {
|
||||
for (const f of readdirSync(join(S, 'korpus', mappe)).sort()) {
|
||||
const innhold = readFileSync(join(S, 'korpus', mappe, f), 'utf8');
|
||||
ut.push({
|
||||
id: /^id:\s*(\S+)/m.exec(innhold)[1],
|
||||
innhold,
|
||||
flatnavn: `${mappe}__${f}`,
|
||||
label: `${mappe}__${f.replace(/\.md$/, '')}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
return ut;
|
||||
}
|
||||
|
||||
function friKontekst(lengde, mal) {
|
||||
const [fm, ...rest] = mal.split('## Fri kontekst');
|
||||
let body = rest.join('## Fri kontekst').trim();
|
||||
body = body.length >= lengde ? body.slice(0, lengde)
|
||||
: body.padEnd(lengde, ` ${body}`).slice(0, lengde);
|
||||
return `${fm}## Fri kontekst\n${body}\n`;
|
||||
}
|
||||
|
||||
function validerInstrument(korpus) {
|
||||
const kanoniske = {};
|
||||
for (const f of ORG_FILES) kanoniske[f] = readFileSync(join(ORG, f), 'utf8');
|
||||
const fri = friKontekst(1216, readFileSync(join(ORG, FREE_CONTEXT_FILE), 'utf8'));
|
||||
const maalt = {
|
||||
a1: [...Object.values(kanoniske), fri, ...korpus.map((d) => d.innhold)].join('\n').length,
|
||||
a2a: buildOrgSummary({ ...kanoniske, [FREE_CONTEXT_FILE]: fri }).length,
|
||||
};
|
||||
return { maalt, forventet: KJENT_GODE, repinnet: REPINNET,
|
||||
avvik: Object.keys(KJENT_GODE).filter((k) => maalt[k] !== KJENT_GODE[k]) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Candidates read from a bundle INDEX ALONE — no concept file is opened.
|
||||
*
|
||||
* `tolkning` is the one free parameter in this measurement, so BOTH settings are
|
||||
* reported rather than one being chosen silently:
|
||||
* 'label' — the label scores at label weight (1), as in the 2026-08-27 B1 arm.
|
||||
* Reproduces B1 for the flat bundle and is the conservative reading.
|
||||
* 'felt' — the label scores as `tittel` (weight 3) and a `sjanger` facet as
|
||||
* `sjanger` (weight 2), i.e. a consumer that trusts the profile's
|
||||
* naming. The reading most generous to OKF.
|
||||
*/
|
||||
function indeksKandidater(bundleDir, tolkning) {
|
||||
const indexMd = readFileSync(join(bundleDir, 'index.md'), 'utf8');
|
||||
const kandidater = [];
|
||||
for (const linje of indexMd.split('\n')) {
|
||||
const m = INDEKS_LINJE.exec(linje.trim());
|
||||
if (!m) continue;
|
||||
const [, label, target, hale] = m;
|
||||
const fasetter = {};
|
||||
for (const del of (hale ?? '').split('; ')) {
|
||||
const kv = FASETT.exec(del.trim());
|
||||
if (kv && kv[2].trim()) fasetter[kv[1]] = kv[2].trim();
|
||||
}
|
||||
const k = {
|
||||
label, target, h2: [],
|
||||
id: ((ID_FRA_STAMME.exec(target) || [])[1] || '').toUpperCase(),
|
||||
body: readFileSync(join(bundleDir, target), 'utf8'),
|
||||
status: fasetter.status,
|
||||
dato: fasetter.dato ?? fasetter.date,
|
||||
};
|
||||
if (tolkning === 'felt') { k.tittel = label; k.sjanger = fasetter.sjanger; }
|
||||
kandidater.push(k);
|
||||
}
|
||||
return { kandidater, listeChars: indexMd.length, scanChars: 0,
|
||||
baererStatus: kandidater.some((c) => c.status !== undefined) };
|
||||
}
|
||||
|
||||
/** Candidates that open every concept's HEAD — the 2026-08-27 B2/B3 arms. */
|
||||
function hodeskannKandidater(kilde) {
|
||||
let scanChars = 0;
|
||||
let listeChars = 0;
|
||||
const kandidater = [];
|
||||
if (kilde.bundleDir) {
|
||||
const indexMd = readFileSync(join(kilde.bundleDir, 'index.md'), 'utf8');
|
||||
listeChars = indexMd.length;
|
||||
for (const linje of indexMd.split('\n')) {
|
||||
const m = INDEKS_LINJE.exec(linje.trim());
|
||||
if (!m) continue;
|
||||
const body = readFileSync(join(kilde.bundleDir, m[2]), 'utf8');
|
||||
scanChars += body.split('\n').slice(0, HEAD_SCAN_LINES + 8).join('\n').length;
|
||||
kandidater.push({ label: m[1], target: m[2], body, h2: [],
|
||||
id: ((ID_FRA_STAMME.exec(m[2]) || [])[1] || '').toUpperCase(), ...headScan(body) });
|
||||
}
|
||||
} else {
|
||||
listeChars = kilde.korpus.map((d) => `${d.flatnavn}\n`).join('').length;
|
||||
for (const d of kilde.korpus) {
|
||||
scanChars += d.innhold.split('\n').slice(0, HEAD_SCAN_LINES + 8).join('\n').length;
|
||||
kandidater.push({ label: d.label, body: d.innhold, h2: [], id: d.id, ...headScan(d.innhold) });
|
||||
}
|
||||
}
|
||||
return { kandidater, listeChars, scanChars, baererStatus: true };
|
||||
}
|
||||
|
||||
function kjoerArm({ navn, kandidater, listeChars, scanChars, baererStatus, gull, budget, k }) {
|
||||
return gull.map((g) => {
|
||||
const qt = tokenize(g.spoersmaal);
|
||||
const scoret = kandidater.map((c) => ({ ...c, score: scoreCandidate(qt, c, tokensMatch) }));
|
||||
const r = cutToBudget({ candidates: scoret, budget, k, applySupersession: baererStatus });
|
||||
const naadd = new Set(r.delivered.map((d) => d.id));
|
||||
return { arm: navn, gull: g.id, klasse: g.klasse,
|
||||
treff: g.fasit.every((id) => naadd.has(id)),
|
||||
felle: g.felle.some((id) => naadd.has(id)),
|
||||
levert: r.delivered.length, agentChars: r.agentChars,
|
||||
indeksChars: listeChars, scanChars,
|
||||
assemblyChars: listeChars + scanChars + r.agentChars };
|
||||
});
|
||||
}
|
||||
|
||||
function oppsummer(rader, flatIndeks) {
|
||||
const klasser = ['enkeltdokument', 'to-dokument', 'stale-fersk', 'distraktor'];
|
||||
return [...new Set(rader.map((r) => r.arm))].map((arm) => {
|
||||
const r = rader.filter((x) => x.arm === arm);
|
||||
const felleR = r.filter((x) => x.klasse === 'stale-fersk' || x.klasse === 'distraktor');
|
||||
const snitt = (f) => Math.round(r.reduce((n, x) => n + f(x), 0) / r.length);
|
||||
return { arm,
|
||||
treff: `${r.filter((x) => x.treff).length}/${r.length}`,
|
||||
...Object.fromEntries(klasser.map((kl) => {
|
||||
const rk = r.filter((x) => x.klasse === kl);
|
||||
return [kl.slice(0, 6), `${rk.filter((x) => x.treff).length}/${rk.length}`];
|
||||
})),
|
||||
feller: `${felleR.filter((x) => x.felle).length}/${felleR.length}`,
|
||||
indeksTegn: r[0].indeksChars,
|
||||
indeksX: `${(r[0].indeksChars / flatIndeks).toFixed(2)}x`,
|
||||
agentTegn: snitt((x) => x.agentChars),
|
||||
assembTegn: snitt((x) => x.assemblyChars) };
|
||||
});
|
||||
}
|
||||
|
||||
function main(argv) {
|
||||
if (!existsSync(S)) { console.error(`Artefaktene mangler: ${S}.`); process.exit(2); }
|
||||
const korpus = lesKorpus();
|
||||
const gull = JSON.parse(readFileSync(join(S, 'gullsett.json'), 'utf8')).items;
|
||||
const val = validerInstrument(korpus);
|
||||
if (val.avvik.length) {
|
||||
console.error('INSTRUMENTET REPRODUSERER IKKE DE KJENT-GODE TALLENE. Stanser.');
|
||||
console.error(JSON.stringify(val, null, 2)); process.exit(1);
|
||||
}
|
||||
|
||||
const PROFILER = [
|
||||
['C flat indeks (DEFAULT)', 'bundle'],
|
||||
['F0 derived alene', 'bundle-f0'],
|
||||
['F1 status+supersedes', 'bundle-tre'],
|
||||
['F2 STRUCTURED_V1 (8)', 'bundle-s1'],
|
||||
['F3 NB (vaare nokler)', 'bundle-nb'],
|
||||
];
|
||||
const flatIndeks = readFileSync(join(S, 'bundle', 'index.md'), 'utf8').length;
|
||||
const alle = [];
|
||||
for (const tolkning of ['label', 'felt']) {
|
||||
for (const [navn, dir] of PROFILER) {
|
||||
const bd = join(S, dir);
|
||||
if (!existsSync(bd)) { console.error(`MANGLER bundle: ${bd}`); process.exit(2); }
|
||||
alle.push(...kjoerArm({ navn, ...indeksKandidater(bd, tolkning), gull,
|
||||
budget: AGENT_BUDGET, k: Infinity }).map((r) => ({ ...r, tolkning })));
|
||||
}
|
||||
}
|
||||
// Referansearmer fra 2026-08-27: hodeskann leser hvert dokuments hode.
|
||||
for (const [navn, kilde] of [
|
||||
['B2 hodeskann (OKF)', { bundleDir: join(S, 'bundle') }],
|
||||
['B3 hodeskann (uten OKF)', { korpus }],
|
||||
]) {
|
||||
alle.push(...kjoerArm({ navn, ...hodeskannKandidater(kilde), gull,
|
||||
budget: AGENT_BUDGET, k: Infinity }).map((r) => ({ ...r, tolkning: 'referanse' })));
|
||||
}
|
||||
|
||||
if (argv.includes('--json')) { console.log(JSON.stringify({ instrument: val, rader: alle }, null, 2)); return; }
|
||||
console.log('INSTRUMENTVALIDERING — korpus-side reprodusert:');
|
||||
console.log(` A1 = ${val.maalt.a1} (forventet ${val.forventet.a1})`);
|
||||
console.log(` A2a = ${val.maalt.a2a} (forventet ${val.forventet.a2a})`);
|
||||
console.log(` RE-PINNET (bundle-side, gammelt anker tapt): hele bundlen ${REPINNET.heleBundlen}, `
|
||||
+ `flat indeks ${REPINNET.indeksFlat}, kilde ${REPINNET.kilde}`);
|
||||
for (const tolkning of ['label', 'felt', 'referanse']) {
|
||||
const r = alle.filter((x) => x.tolkning === tolkning);
|
||||
if (!r.length) continue;
|
||||
console.log(`\n=== TOLKNING: ${tolkning} (budsjett ${AGENT_BUDGET}, nevner 40, felle-nevner 20) ===`);
|
||||
console.table(oppsummer(r, flatIndeks));
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) main(process.argv.slice(2));
|
||||
Loading…
Add table
Add a link
Reference in a new issue