ms-ai-architect/scripts/kb-eval/measure-okf-consume.mjs

#!/usr/bin/env node
// measure-okf-consume.mjs — EVAL DRIVER for the OKF bundle consumption skill.
//
// Measures the skill's deterministic cut against the SAME corpus, gold set and
// instrument as the 2026-08 bake-off, so the numbers are comparable.
//
// This is eval code. It is not read by an agent and nothing in the plugin's
// runtime imports it; the skill imports scripts/okf-consume/, not this file.
//
// Run:  node scripts/kb-eval/measure-okf-consume.mjs [--json]
// Needs the bake-off artefacts at ~/okf-bakeoff-2026-08/ (see its README).

import { readFileSync, readdirSync, existsSync } from 'node:fs';
import { join } from 'node:path';
import { homedir } from 'node:os';
import {
  tokenize, tokensMatch, tokensMatchCompound, parseIndex, headScan,
  scoreCandidate, cutToBudget, AGENT_BUDGET, HEAD_SCAN_LINES,
} from '../okf-consume/lib/bundle-cut.mjs';
import { buildOrgSummary, ORG_FILES, FREE_CONTEXT_FILE }
  from '../kb-update/lib/user-data.mjs';

const S = join(homedir(), 'okf-bakeoff-2026-08');
const ORG = join(homedir(), '.claude', 'ms-ai-architect', 'org');
const MAPPER = ['arkitektur', 'hendelser', 'leverandor', 'styring'];

// --- instrumentvalidering (C6): reproduser kjent-gode tall FOER egne tall brukes ---

// Amendement 2: A2b (3412) og A3 (5879) fra bake-off-dokumentet reproduserer IKKE
// fra de reddede artefaktene -- korpuset er verifisert uendret (55/55 sha256), og
// verken HEAD eller e4f1642 gir 3412. Triplet er byttet til de tre som faktisk
// reproduserer, og avviket rapporteres i stedet for aa brukes som referanse.
const KJENT_GODE = { a1: 31343, a2a: 2884, heleBundlen: 45580 };

function lesKorpus() {
  const ut = [];
  for (const mappe of MAPPER) {
    for (const f of readdirSync(join(S, 'korpus', mappe)).sort()) {
      const innhold = readFileSync(join(S, 'korpus', mappe, f), 'utf8');
      ut.push({
        id: /^id:\s*(\S+)/m.exec(innhold)[1],
        mappe, fil: f, innhold,
        flatnavn: `${mappe}__${f}`,
        label: `${mappe}__${f.replace(/\.md$/, '')}`,
      });
    }
  }
  return ut;
}

function friKontekst(lengde, mal) {
  const [fm, ...rest] = mal.split('## Fri kontekst');
  let body = rest.join('## Fri kontekst').trim();
  body = body.length >= lengde ? body.slice(0, lengde)
    : body.padEnd(lengde, ` ${body}`).slice(0, lengde);
  return `${fm}## Fri kontekst\n${body}\n`;
}

function validerInstrument(korpus) {
  const kanoniske = {};
  for (const f of ORG_FILES) kanoniske[f] = readFileSync(join(ORG, f), 'utf8');
  const fri = friKontekst(1216, readFileSync(join(ORG, FREE_CONTEXT_FILE), 'utf8'));

  const a1 = [...Object.values(kanoniske), fri, ...korpus.map((d) => d.innhold)].join('\n').length;

  // A2a: dagens ikke-rekursive mekanisme -- bare de kanoniske org-filene naar fram.
  const a2a = buildOrgSummary({ ...kanoniske, [FREE_CONTEXT_FILE]: fri }).length;

  const bundleDir = join(S, 'bundle');
  const heleBundlen = readdirSync(bundleDir)
    .filter((f) => f.endsWith('.md'))
    .reduce((n, f) => n + readFileSync(join(bundleDir, f), 'utf8').length, 0);

  const maalt = { a1, a2a, heleBundlen };
  const avvik = Object.keys(KJENT_GODE).filter((k) => maalt[k] !== KJENT_GODE[k]);
  return { maalt, forventet: KJENT_GODE, avvik };
}

// --- kandidatbygging ---

const ID_FRA_LABEL = /__([A-ZÆØÅ]\d{2})-/;

function bundleKandidater(bundleDir, mode) {
  const indexMd = readFileSync(join(bundleDir, 'index.md'), 'utf8');
  const entries = parseIndex(indexMd);
  let scanChars = 0;
  const kandidater = entries.map(({ label, target }) => {
    const body = readFileSync(join(bundleDir, target), 'utf8');
    const k = { label, target, body, h2: [], id: (ID_FRA_LABEL.exec(label) || [])[1] };
    if (mode === 'headscan') {
      scanChars += body.split('\n').slice(0, HEAD_SCAN_LINES + 8).join('\n').length;
      Object.assign(k, headScan(body));
    }
    return k;
  });
  return { kandidater, listeChars: indexMd.length, scanChars };
}

function reneKandidater(korpus, mode) {
  // Ingen OKF-profil paastaar noe om denne katalogen, saa en listing er lovlig her
  // (sjekkliste punkt 9 binder bare der en profil sier indeksen er forfattet).
  const listeChars = korpus.map((d) => `${d.flatnavn}\n`).join('').length;
  let scanChars = 0;
  const kandidater = korpus.map((d) => {
    const k = { label: d.label, body: d.innhold, h2: [], id: d.id };
    if (mode === 'headscan') {
      scanChars += d.innhold.split('\n').slice(0, HEAD_SCAN_LINES + 8).join('\n').length;
      Object.assign(k, headScan(d.innhold));
    }
    return k;
  });
  return { kandidater, listeChars, scanChars };
}

// --- kjoer én arm over hele gullsettet ---

function kjoerArm({ navn, kandidater, listeChars, scanChars, mode, matcher, gull, k, budget }) {
  const rader = [];
  for (const g of gull) {
    const qt = tokenize(g.spoersmaal);
    const scoret = kandidater.map((c) => ({ ...c, score: scoreCandidate(qt, c, matcher) }));
    const r = cutToBudget({
      candidates: scoret, budget, k, applySupersession: mode === 'headscan',
    });
    const naadd = new Set(r.delivered.map((d) => d.id));
    rader.push({
      arm: navn, gull: g.id, klasse: g.klasse,
      treff: g.fasit.every((id) => naadd.has(id)),
      felle: g.felle.some((id) => naadd.has(id)),
      levert: r.delivered.length,
      unread: r.unread,
      denominator: r.denominator,
      superseded: r.superseded,
      statusUnknown: r.statusUnknown,
      agentChars: r.agentChars,
      assemblyChars: listeChars + scanChars + r.agentChars,
    });
  }
  return rader;
}

function oppsummer(rader) {
  const klasser = ['enkeltdokument', 'to-dokument', 'stale-fersk', 'distraktor'];
  const armer = [...new Set(rader.map((r) => r.arm))];
  return armer.map((arm) => {
    const r = rader.filter((x) => x.arm === arm);
    const felleR = r.filter((x) => x.klasse === 'stale-fersk' || x.klasse === 'distraktor');
    const snitt = (f) => Math.round(r.reduce((n, x) => n + f(x), 0) / r.length);
    return {
      arm,
      treff: `${r.filter((x) => x.treff).length}/${r.length}`,
      perKlasse: Object.fromEntries(klasser.map((kl) => {
        const rk = r.filter((x) => x.klasse === kl);
        return [kl, `${rk.filter((x) => x.treff).length}/${rk.length}`];
      })),
      felleEksponering: `${felleR.filter((x) => x.felle).length}/${felleR.length}`,
      snittAgentTegn: snitt((x) => x.agentChars),
      snittAssembleringTegn: snitt((x) => x.assemblyChars),
      snittLevert: (r.reduce((n, x) => n + x.levert, 0) / r.length).toFixed(1),
      snittUnread: `${(r.reduce((n, x) => n + x.unread, 0) / r.length).toFixed(1)}/${r[0].denominator}`,
    };
  });
}

function main(argv) {
  if (!existsSync(S)) {
    console.error(`Artefaktene mangler: ${S}. Se repoets STATE / bake-off-README.`);
    process.exit(2);
  }
  const korpus = lesKorpus();
  const gull = JSON.parse(readFileSync(join(S, 'gullsett.json'), 'utf8')).items;

  const val = validerInstrument(korpus);
  if (val.avvik.length) {
    console.error('INSTRUMENTET REPRODUSERER IKKE DE KJENT-GODE TALLENE. Stanser.');
    console.error(JSON.stringify(val, null, 2));
    process.exit(1);
  }

  const bundleDir = join(S, 'bundle');
  const bIdx = bundleKandidater(bundleDir, 'index');
  const bHead = bundleKandidater(bundleDir, 'headscan');
  const rHead = reneKandidater(korpus, 'headscan');

  const armer = [
    { navn: 'B1  indeks (prereg)', ...bIdx, mode: 'index', matcher: tokensMatch },
    { navn: 'B2  hodeskann (prereg)', ...bHead, mode: 'headscan', matcher: tokensMatch },
    { navn: 'B3  rene filer (prereg)', ...rHead, mode: 'headscan', matcher: tokensMatch },
    { navn: 'B2+ hodeskann (post-hoc)', ...bHead, mode: 'headscan', matcher: tokensMatchCompound },
    { navn: 'B3+ rene filer (post-hoc)', ...rHead, mode: 'headscan', matcher: tokensMatchCompound },
  ];

  const alle = [];
  for (const budsjett of [{ k: Infinity, budget: AGENT_BUDGET, merk: 'budsjett 12000' },
    { k: 3, budget: AGENT_BUDGET, merk: 'k=3' }]) {
    for (const a of armer) {
      alle.push(...kjoerArm({ ...a, gull, k: budsjett.k, budget: budsjett.budget })
        .map((r) => ({ ...r, regime: budsjett.merk })));
    }
  }

  if (argv.includes('--json')) {
    console.log(JSON.stringify({ instrument: val, rader: alle }, null, 2));
    return;
  }

  console.log('INSTRUMENTVALIDERING (C6) — kjent-gode tall reprodusert:');
  console.log(`  A1 last-alt n=55 : ${val.maalt.a1} (forventet ${val.forventet.a1})`);
  console.log(`  A2a n=55         : ${val.maalt.a2a} (forventet ${val.forventet.a2a})`);
  console.log(`  Hele bundlen     : ${val.maalt.heleBundlen} (forventet ${val.forventet.heleBundlen})`);
  for (const regime of ['budsjett 12000', 'k=3']) {
    console.log(`\n=== REGIME: ${regime} (nevner 40 gullspoersmaal, felle-nevner 20) ===`);
    console.table(oppsummer(alle.filter((r) => r.regime === regime)));
  }
}

if (import.meta.url === `file://${process.argv[1]}`) main(process.argv.slice(2));