ms-ai-architect/scripts/kb-eval/measure-okf-facets.mjs

#!/usr/bin/env node
// measure-okf-facets.mjs — EVAL DRIVER for the STRUCTURED_V1 facet bake-off.
//
// Answers order 20260827T084938Z-4734427711: is a FACETED OKF index worth its
// character cost on THIS corpus, against THESE 40 gold questions?
//
// This is eval code, exactly like measure-okf-consume.mjs. No agent reads it and
// nothing in the plugin runtime imports it; it imports scripts/okf-consume/,
// never the other way round. No production behaviour changes with this file.
//
// Run:  node scripts/kb-eval/measure-okf-facets.mjs [--json]
// Needs ~/okf-bakeoff-2026-08/ with the bundles built by bygg/build-bundles.py.

import { readFileSync, readdirSync, existsSync } from 'node:fs';
import { join } from 'node:path';
import { homedir } from 'node:os';
import {
  tokenize, tokensMatch, headScan, scoreCandidate, cutToBudget,
  AGENT_BUDGET, HEAD_SCAN_LINES,
} from '../okf-consume/lib/bundle-cut.mjs';
import { buildOrgSummary, ORG_FILES, FREE_CONTEXT_FILE } from '../kb-update/lib/user-data.mjs';

// Measurement root. Overridable so a RE-MEASUREMENT builds into a fresh tree
// instead of over the 2026-08-27 artefacts: build-bundles.py rmtree's its
// output dirs, and that is exactly how the heleBundlen=45580 anchor was lost.
const S = process.env.OKF_MAALEROT || join(homedir(), 'okf-bakeoff-2026-08');
const ORG = join(homedir(), '.claude', 'ms-ai-architect', 'org');
const MAPPER = ['arkitektur', 'hendelser', 'leverandor', 'styring'];

// Instrument gate. The two CORPUS-side anchors from the 2026-08-27 consume
// measurement, which are what this measurement actually rests on. The bundle-side
// anchor (heleBundlen 45580) is NOT here: this session rebuilt the control bundle
// with llm-ingestion-okf @ d35bcb2 and lost the 01e4170 artefact, so the old value
// is unreproducible. The new value is re-pinned below and the loss is reported.
const KJENT_GODE = { a1: 31343, a2a: 2884 };
const REPINNET = { heleBundlen: 45898, indeksFlat: 4521, kilde: 'llm-ingestion-okf @ d35bcb2' };

// The index line grammar STRUCTURED_V1 renders: DEFAULT's link plus a facet tail.
const INDEKS_LINJE = /^- \[([^\]]*)\]\(([^)]+)\)(?: — (.*))?$/;
const FASETT = /^([a-zA-Z_æøåÆØÅ]+):\s*(.+)$/;
const ID_FRA_STAMME = /-([a-z]\d{2})-/;

function lesKorpus() {
  const ut = [];
  for (const mappe of MAPPER) {
    for (const f of readdirSync(join(S, 'korpus', mappe)).sort()) {
      const innhold = readFileSync(join(S, 'korpus', mappe, f), 'utf8');
      ut.push({
        id: /^id:\s*(\S+)/m.exec(innhold)[1],
        innhold,
        flatnavn: `${mappe}__${f}`,
        label: `${mappe}__${f.replace(/\.md$/, '')}`,
      });
    }
  }
  return ut;
}

function friKontekst(lengde, mal) {
  const [fm, ...rest] = mal.split('## Fri kontekst');
  let body = rest.join('## Fri kontekst').trim();
  body = body.length >= lengde ? body.slice(0, lengde)
    : body.padEnd(lengde, ` ${body}`).slice(0, lengde);
  return `${fm}## Fri kontekst\n${body}\n`;
}

function validerInstrument(korpus) {
  const kanoniske = {};
  for (const f of ORG_FILES) kanoniske[f] = readFileSync(join(ORG, f), 'utf8');
  const fri = friKontekst(1216, readFileSync(join(ORG, FREE_CONTEXT_FILE), 'utf8'));
  const maalt = {
    a1: [...Object.values(kanoniske), fri, ...korpus.map((d) => d.innhold)].join('\n').length,
    a2a: buildOrgSummary({ ...kanoniske, [FREE_CONTEXT_FILE]: fri }).length,
  };
  return { maalt, forventet: KJENT_GODE, repinnet: REPINNET,
    avvik: Object.keys(KJENT_GODE).filter((k) => maalt[k] !== KJENT_GODE[k]) };
}

/**
 * Candidates read from a bundle INDEX ALONE — no concept file is opened.
 *
 * `tolkning` is the one free parameter in this measurement, so BOTH settings are
 * reported rather than one being chosen silently:
 *   'label'  — the label scores at label weight (1), as in the 2026-08-27 B1 arm.
 *              Reproduces B1 for the flat bundle and is the conservative reading.
 *   'felt'   — the label scores as `tittel` (weight 3) and a `sjanger` facet as
 *              `sjanger` (weight 2), i.e. a consumer that trusts the profile's
 *              naming. The reading most generous to OKF.
 */
function indeksKandidater(bundleDir, tolkning) {
  const indexMd = readFileSync(join(bundleDir, 'index.md'), 'utf8');
  const kandidater = [];
  for (const linje of indexMd.split('\n')) {
    const m = INDEKS_LINJE.exec(linje.trim());
    if (!m) continue;
    const [, label, target, hale] = m;
    const fasetter = {};
    for (const del of (hale ?? '').split('; ')) {
      const kv = FASETT.exec(del.trim());
      if (kv && kv[2].trim()) fasetter[kv[1]] = kv[2].trim();
    }
    const k = {
      label, target, h2: [],
      id: ((ID_FRA_STAMME.exec(target) || [])[1] || '').toUpperCase(),
      body: readFileSync(join(bundleDir, target), 'utf8'),
      status: fasetter.status,
      dato: fasetter.dato ?? fasetter.date,
    };
    if (tolkning === 'felt') { k.tittel = label; k.sjanger = fasetter.sjanger; }
    kandidater.push(k);
  }
  return { kandidater, listeChars: indexMd.length, scanChars: 0,
    baererStatus: kandidater.some((c) => c.status !== undefined) };
}

/** Candidates that open every concept's HEAD — the 2026-08-27 B2/B3 arms. */
function hodeskannKandidater(kilde) {
  let scanChars = 0;
  let listeChars = 0;
  const kandidater = [];
  if (kilde.bundleDir) {
    const indexMd = readFileSync(join(kilde.bundleDir, 'index.md'), 'utf8');
    listeChars = indexMd.length;
    for (const linje of indexMd.split('\n')) {
      const m = INDEKS_LINJE.exec(linje.trim());
      if (!m) continue;
      const body = readFileSync(join(kilde.bundleDir, m[2]), 'utf8');
      scanChars += body.split('\n').slice(0, HEAD_SCAN_LINES + 8).join('\n').length;
      kandidater.push({ label: m[1], target: m[2], body, h2: [],
        id: ((ID_FRA_STAMME.exec(m[2]) || [])[1] || '').toUpperCase(), ...headScan(body) });
    }
  } else {
    listeChars = kilde.korpus.map((d) => `${d.flatnavn}\n`).join('').length;
    for (const d of kilde.korpus) {
      scanChars += d.innhold.split('\n').slice(0, HEAD_SCAN_LINES + 8).join('\n').length;
      kandidater.push({ label: d.label, body: d.innhold, h2: [], id: d.id, ...headScan(d.innhold) });
    }
  }
  return { kandidater, listeChars, scanChars, baererStatus: true };
}

function kjoerArm({ navn, kandidater, listeChars, scanChars, baererStatus, gull, budget, k }) {
  return gull.map((g) => {
    const qt = tokenize(g.spoersmaal);
    const scoret = kandidater.map((c) => ({ ...c, score: scoreCandidate(qt, c, tokensMatch) }));
    const r = cutToBudget({ candidates: scoret, budget, k, applySupersession: baererStatus });
    const naadd = new Set(r.delivered.map((d) => d.id));
    return { arm: navn, gull: g.id, klasse: g.klasse,
      treff: g.fasit.every((id) => naadd.has(id)),
      felle: g.felle.some((id) => naadd.has(id)),
      levert: r.delivered.length, agentChars: r.agentChars,
      indeksChars: listeChars, scanChars,
      assemblyChars: listeChars + scanChars + r.agentChars };
  });
}

function oppsummer(rader, flatIndeks) {
  const klasser = ['enkeltdokument', 'to-dokument', 'stale-fersk', 'distraktor'];
  return [...new Set(rader.map((r) => r.arm))].map((arm) => {
    const r = rader.filter((x) => x.arm === arm);
    const felleR = r.filter((x) => x.klasse === 'stale-fersk' || x.klasse === 'distraktor');
    const snitt = (f) => Math.round(r.reduce((n, x) => n + f(x), 0) / r.length);
    return { arm,
      treff: `${r.filter((x) => x.treff).length}/${r.length}`,
      ...Object.fromEntries(klasser.map((kl) => {
        const rk = r.filter((x) => x.klasse === kl);
        return [kl.slice(0, 6), `${rk.filter((x) => x.treff).length}/${rk.length}`];
      })),
      feller: `${felleR.filter((x) => x.felle).length}/${felleR.length}`,
      indeksTegn: r[0].indeksChars,
      indeksX: `${(r[0].indeksChars / flatIndeks).toFixed(2)}x`,
      agentTegn: snitt((x) => x.agentChars),
      assembTegn: snitt((x) => x.assemblyChars) };
  });
}

function main(argv) {
  if (!existsSync(S)) { console.error(`Artefaktene mangler: ${S}.`); process.exit(2); }
  const korpus = lesKorpus();
  const gull = JSON.parse(readFileSync(join(S, 'gullsett.json'), 'utf8')).items;
  const val = validerInstrument(korpus);
  if (val.avvik.length) {
    console.error('INSTRUMENTET REPRODUSERER IKKE DE KJENT-GODE TALLENE. Stanser.');
    console.error(JSON.stringify(val, null, 2)); process.exit(1);
  }

  const PROFILER = [
    ['C   flat indeks (DEFAULT)', 'bundle'],
    ['F0  derived alene', 'bundle-f0'],
    ['F1  status+supersedes', 'bundle-tre'],
    ['F2  STRUCTURED_V1 (8)', 'bundle-s1'],
    ['F3  NB (vaare nokler)', 'bundle-nb'],
  ];
  const flatIndeks = readFileSync(join(S, 'bundle', 'index.md'), 'utf8').length;
  const alle = [];
  for (const tolkning of ['label', 'felt']) {
    for (const [navn, dir] of PROFILER) {
      const bd = join(S, dir);
      if (!existsSync(bd)) { console.error(`MANGLER bundle: ${bd}`); process.exit(2); }
      alle.push(...kjoerArm({ navn, ...indeksKandidater(bd, tolkning), gull,
        budget: AGENT_BUDGET, k: Infinity }).map((r) => ({ ...r, tolkning })));
    }
  }
  // Referansearmer fra 2026-08-27: hodeskann leser hvert dokuments hode.
  for (const [navn, kilde] of [
    ['B2  hodeskann (OKF)', { bundleDir: join(S, 'bundle') }],
    ['B3  hodeskann (uten OKF)', { korpus }],
  ]) {
    alle.push(...kjoerArm({ navn, ...hodeskannKandidater(kilde), gull,
      budget: AGENT_BUDGET, k: Infinity }).map((r) => ({ ...r, tolkning: 'referanse' })));
  }

  if (argv.includes('--json')) { console.log(JSON.stringify({ instrument: val, rader: alle }, null, 2)); return; }
  console.log('INSTRUMENTVALIDERING — korpus-side reprodusert:');
  console.log(`  A1  = ${val.maalt.a1} (forventet ${val.forventet.a1})`);
  console.log(`  A2a = ${val.maalt.a2a} (forventet ${val.forventet.a2a})`);
  console.log(`  RE-PINNET (bundle-side, gammelt anker tapt): hele bundlen ${REPINNET.heleBundlen}, `
    + `flat indeks ${REPINNET.indeksFlat}, kilde ${REPINNET.kilde}`);
  for (const tolkning of ['label', 'felt', 'referanse']) {
    const r = alle.filter((x) => x.tolkning === tolkning);
    if (!r.length) continue;
    console.log(`\n=== TOLKNING: ${tolkning} (budsjett ${AGENT_BUDGET}, nevner 40, felle-nevner 20) ===`);
    console.table(oppsummer(r, flatIndeks));
  }
}

if (import.meta.url === `file://${process.argv[1]}`) main(process.argv.slice(2));