#!/usr/bin/env node // measure-okf-facets.mjs — EVAL DRIVER for the STRUCTURED_V1 facet bake-off. // // Answers order 20260827T084938Z-4734427711: is a FACETED OKF index worth its // character cost on THIS corpus, against THESE 40 gold questions? // // This is eval code, exactly like measure-okf-consume.mjs. No agent reads it and // nothing in the plugin runtime imports it; it imports scripts/okf-consume/, // never the other way round. No production behaviour changes with this file. // // Run: node scripts/kb-eval/measure-okf-facets.mjs [--json] // Needs ~/okf-bakeoff-2026-08/ with the bundles built by bygg/build-bundles.py. import { readFileSync, readdirSync, existsSync } from 'node:fs'; import { join } from 'node:path'; import { homedir } from 'node:os'; import { tokenize, tokensMatch, headScan, scoreCandidate, cutToBudget, AGENT_BUDGET, HEAD_SCAN_LINES, } from '../okf-consume/lib/bundle-cut.mjs'; import { buildOrgSummary, ORG_FILES, FREE_CONTEXT_FILE } from '../kb-update/lib/user-data.mjs'; // Measurement root. Overridable so a RE-MEASUREMENT builds into a fresh tree // instead of over the 2026-08-27 artefacts: build-bundles.py rmtree's its // output dirs, and that is exactly how the heleBundlen=45580 anchor was lost. const S = process.env.OKF_MAALEROT || join(homedir(), 'okf-bakeoff-2026-08'); const ORG = join(homedir(), '.claude', 'ms-ai-architect', 'org'); const MAPPER = ['arkitektur', 'hendelser', 'leverandor', 'styring']; // Instrument gate. The two CORPUS-side anchors from the 2026-08-27 consume // measurement, which are what this measurement actually rests on. The bundle-side // anchor (heleBundlen 45580) is NOT here: this session rebuilt the control bundle // with llm-ingestion-okf @ d35bcb2 and lost the 01e4170 artefact, so the old value // is unreproducible. The new value is re-pinned below and the loss is reported. const KJENT_GODE = { a1: 31343, a2a: 2884 }; const REPINNET = { heleBundlen: 45898, indeksFlat: 4521, kilde: 'llm-ingestion-okf @ d35bcb2' }; // The index line grammar STRUCTURED_V1 renders: DEFAULT's link plus a facet tail. const INDEKS_LINJE = /^- \[([^\]]*)\]\(([^)]+)\)(?: — (.*))?$/; const FASETT = /^([a-zA-Z_æøåÆØÅ]+):\s*(.+)$/; const ID_FRA_STAMME = /-([a-z]\d{2})-/; function lesKorpus() { const ut = []; for (const mappe of MAPPER) { for (const f of readdirSync(join(S, 'korpus', mappe)).sort()) { const innhold = readFileSync(join(S, 'korpus', mappe, f), 'utf8'); ut.push({ id: /^id:\s*(\S+)/m.exec(innhold)[1], innhold, flatnavn: `${mappe}__${f}`, label: `${mappe}__${f.replace(/\.md$/, '')}`, }); } } return ut; } function friKontekst(lengde, mal) { const [fm, ...rest] = mal.split('## Fri kontekst'); let body = rest.join('## Fri kontekst').trim(); body = body.length >= lengde ? body.slice(0, lengde) : body.padEnd(lengde, ` ${body}`).slice(0, lengde); return `${fm}## Fri kontekst\n${body}\n`; } function validerInstrument(korpus) { const kanoniske = {}; for (const f of ORG_FILES) kanoniske[f] = readFileSync(join(ORG, f), 'utf8'); const fri = friKontekst(1216, readFileSync(join(ORG, FREE_CONTEXT_FILE), 'utf8')); const maalt = { a1: [...Object.values(kanoniske), fri, ...korpus.map((d) => d.innhold)].join('\n').length, a2a: buildOrgSummary({ ...kanoniske, [FREE_CONTEXT_FILE]: fri }).length, }; return { maalt, forventet: KJENT_GODE, repinnet: REPINNET, avvik: Object.keys(KJENT_GODE).filter((k) => maalt[k] !== KJENT_GODE[k]) }; } /** * Candidates read from a bundle INDEX ALONE — no concept file is opened. * * `tolkning` is the one free parameter in this measurement, so BOTH settings are * reported rather than one being chosen silently: * 'label' — the label scores at label weight (1), as in the 2026-08-27 B1 arm. * Reproduces B1 for the flat bundle and is the conservative reading. * 'felt' — the label scores as `tittel` (weight 3) and a `sjanger` facet as * `sjanger` (weight 2), i.e. a consumer that trusts the profile's * naming. The reading most generous to OKF. */ function indeksKandidater(bundleDir, tolkning) { const indexMd = readFileSync(join(bundleDir, 'index.md'), 'utf8'); const kandidater = []; for (const linje of indexMd.split('\n')) { const m = INDEKS_LINJE.exec(linje.trim()); if (!m) continue; const [, label, target, hale] = m; const fasetter = {}; for (const del of (hale ?? '').split('; ')) { const kv = FASETT.exec(del.trim()); if (kv && kv[2].trim()) fasetter[kv[1]] = kv[2].trim(); } const k = { label, target, h2: [], id: ((ID_FRA_STAMME.exec(target) || [])[1] || '').toUpperCase(), body: readFileSync(join(bundleDir, target), 'utf8'), status: fasetter.status, dato: fasetter.dato ?? fasetter.date, }; if (tolkning === 'felt') { k.tittel = label; k.sjanger = fasetter.sjanger; } kandidater.push(k); } return { kandidater, listeChars: indexMd.length, scanChars: 0, baererStatus: kandidater.some((c) => c.status !== undefined) }; } /** Candidates that open every concept's HEAD — the 2026-08-27 B2/B3 arms. */ function hodeskannKandidater(kilde) { let scanChars = 0; let listeChars = 0; const kandidater = []; if (kilde.bundleDir) { const indexMd = readFileSync(join(kilde.bundleDir, 'index.md'), 'utf8'); listeChars = indexMd.length; for (const linje of indexMd.split('\n')) { const m = INDEKS_LINJE.exec(linje.trim()); if (!m) continue; const body = readFileSync(join(kilde.bundleDir, m[2]), 'utf8'); scanChars += body.split('\n').slice(0, HEAD_SCAN_LINES + 8).join('\n').length; kandidater.push({ label: m[1], target: m[2], body, h2: [], id: ((ID_FRA_STAMME.exec(m[2]) || [])[1] || '').toUpperCase(), ...headScan(body) }); } } else { listeChars = kilde.korpus.map((d) => `${d.flatnavn}\n`).join('').length; for (const d of kilde.korpus) { scanChars += d.innhold.split('\n').slice(0, HEAD_SCAN_LINES + 8).join('\n').length; kandidater.push({ label: d.label, body: d.innhold, h2: [], id: d.id, ...headScan(d.innhold) }); } } return { kandidater, listeChars, scanChars, baererStatus: true }; } function kjoerArm({ navn, kandidater, listeChars, scanChars, baererStatus, gull, budget, k }) { return gull.map((g) => { const qt = tokenize(g.spoersmaal); const scoret = kandidater.map((c) => ({ ...c, score: scoreCandidate(qt, c, tokensMatch) })); const r = cutToBudget({ candidates: scoret, budget, k, applySupersession: baererStatus }); const naadd = new Set(r.delivered.map((d) => d.id)); return { arm: navn, gull: g.id, klasse: g.klasse, treff: g.fasit.every((id) => naadd.has(id)), felle: g.felle.some((id) => naadd.has(id)), levert: r.delivered.length, agentChars: r.agentChars, indeksChars: listeChars, scanChars, assemblyChars: listeChars + scanChars + r.agentChars }; }); } function oppsummer(rader, flatIndeks) { const klasser = ['enkeltdokument', 'to-dokument', 'stale-fersk', 'distraktor']; return [...new Set(rader.map((r) => r.arm))].map((arm) => { const r = rader.filter((x) => x.arm === arm); const felleR = r.filter((x) => x.klasse === 'stale-fersk' || x.klasse === 'distraktor'); const snitt = (f) => Math.round(r.reduce((n, x) => n + f(x), 0) / r.length); return { arm, treff: `${r.filter((x) => x.treff).length}/${r.length}`, ...Object.fromEntries(klasser.map((kl) => { const rk = r.filter((x) => x.klasse === kl); return [kl.slice(0, 6), `${rk.filter((x) => x.treff).length}/${rk.length}`]; })), feller: `${felleR.filter((x) => x.felle).length}/${felleR.length}`, indeksTegn: r[0].indeksChars, indeksX: `${(r[0].indeksChars / flatIndeks).toFixed(2)}x`, agentTegn: snitt((x) => x.agentChars), assembTegn: snitt((x) => x.assemblyChars) }; }); } function main(argv) { if (!existsSync(S)) { console.error(`Artefaktene mangler: ${S}.`); process.exit(2); } const korpus = lesKorpus(); const gull = JSON.parse(readFileSync(join(S, 'gullsett.json'), 'utf8')).items; const val = validerInstrument(korpus); if (val.avvik.length) { console.error('INSTRUMENTET REPRODUSERER IKKE DE KJENT-GODE TALLENE. Stanser.'); console.error(JSON.stringify(val, null, 2)); process.exit(1); } const PROFILER = [ ['C flat indeks (DEFAULT)', 'bundle'], ['F0 derived alene', 'bundle-f0'], ['F1 status+supersedes', 'bundle-tre'], ['F2 STRUCTURED_V1 (8)', 'bundle-s1'], ['F3 NB (vaare nokler)', 'bundle-nb'], ]; const flatIndeks = readFileSync(join(S, 'bundle', 'index.md'), 'utf8').length; const alle = []; for (const tolkning of ['label', 'felt']) { for (const [navn, dir] of PROFILER) { const bd = join(S, dir); if (!existsSync(bd)) { console.error(`MANGLER bundle: ${bd}`); process.exit(2); } alle.push(...kjoerArm({ navn, ...indeksKandidater(bd, tolkning), gull, budget: AGENT_BUDGET, k: Infinity }).map((r) => ({ ...r, tolkning }))); } } // Referansearmer fra 2026-08-27: hodeskann leser hvert dokuments hode. for (const [navn, kilde] of [ ['B2 hodeskann (OKF)', { bundleDir: join(S, 'bundle') }], ['B3 hodeskann (uten OKF)', { korpus }], ]) { alle.push(...kjoerArm({ navn, ...hodeskannKandidater(kilde), gull, budget: AGENT_BUDGET, k: Infinity }).map((r) => ({ ...r, tolkning: 'referanse' }))); } if (argv.includes('--json')) { console.log(JSON.stringify({ instrument: val, rader: alle }, null, 2)); return; } console.log('INSTRUMENTVALIDERING — korpus-side reprodusert:'); console.log(` A1 = ${val.maalt.a1} (forventet ${val.forventet.a1})`); console.log(` A2a = ${val.maalt.a2a} (forventet ${val.forventet.a2a})`); console.log(` RE-PINNET (bundle-side, gammelt anker tapt): hele bundlen ${REPINNET.heleBundlen}, ` + `flat indeks ${REPINNET.indeksFlat}, kilde ${REPINNET.kilde}`); for (const tolkning of ['label', 'felt', 'referanse']) { const r = alle.filter((x) => x.tolkning === tolkning); if (!r.length) continue; console.log(`\n=== TOLKNING: ${tolkning} (budsjett ${AGENT_BUDGET}, nevner 40, felle-nevner 20) ===`); console.table(oppsummer(r, flatIndeks)); } } if (import.meta.url === `file://${process.argv[1]}`) main(process.argv.slice(2));