open/ms-ai-architect
1
0
Fork 0
Code Issues Releases 1 Activity

feat(ms-ai-architect): OKF-konsum-cutter mot konsum-kontraktens §4, målt [skip-docs]

Browse source 
Deterministisk halvdel av konsum-skillen (ordre 20260826T123330Z-1842820111):
tokenisering, feltvekting, mekanisk supersesjonsregel og budsjett-kutt med
[unread]-kvittering mot hele kandidatnevneren. Eval-driver mot bake-off-korpuset.

Målt, budsjett 12 000 tegn, nevner 40 gullspørsmål:
  B1 kun indekslabel        6/40   feller 4/20
  B2 konform strategi      12/40   feller 0/20
  B3 kontroll uten OKF     12/40   feller 1/20
  B2+/B3+ (POST-HOC)       13/40 / 14/40

B2 == B3 er R3, den ene forhåndsregistrerte sammenligningen: med denne rangereren
gir OKF-metadataen ingen målbar gevinst på primærmetrikken. Differansen mot
bake-offens A3 (40/40, forfatter-informert velger = øvre grense) tilskrives per
låst regel R5 RANGEREREN, ikke OKF som teknologi.

Instrumentvalidering reproduserer eksakt: A1=31343, A2a=2884, hele bundlen=45580.
Suite 1097/1097 (1070 sporet + 27 nye). Negativ kontroll kjørt: tre mutasjoner i
bundle-cut.mjs, alle drept (1/1/3 feil), treet restaurert byte-identisk.

Konformitet mot §4: 8 av 10 oppfylt, punkt 3 delvis (falsifisering ikke skrevet
før kjøring), punkt 5 ikke oppfylt (markeringssett ikke deklarert).

[skip-docs]: ren intern måle- og evalueringsmekanikk. Ingen ny kommando, agent
eller skill; ingen brukervendt flate endret, så README/CLAUDE.md er uberørt.
This commit is contained in:
Kjell Tore Guttormsen 2026-08-27 12:13:03 +02:00
commit c0f10618da
4 changed files with 785 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

217
scripts/kb-eval/measure-okf-consume.mjs Normal file
View file

@ -0,0 +1,217 @@
#!/usr/bin/env node
// measure-okf-consume.mjs — EVAL DRIVER for the OKF bundle consumption skill.
//
// Measures the skill's deterministic cut against the SAME corpus, gold set and
// instrument as the 2026-08 bake-off, so the numbers are comparable.
//
// This is eval code. It is not read by an agent and nothing in the plugin's
// runtime imports it; the skill imports scripts/okf-consume/, not this file.
//
// Run: node scripts/kb-eval/measure-okf-consume.mjs [--json]
// Needs the bake-off artefacts at ~/okf-bakeoff-2026-08/ (see its README).
import { readFileSync, readdirSync, existsSync } from 'node:fs';
import { join } from 'node:path';
import { homedir } from 'node:os';
import {
tokenize, tokensMatch, tokensMatchCompound, parseIndex, headScan,
scoreCandidate, cutToBudget, AGENT_BUDGET, HEAD_SCAN_LINES,
} from '../okf-consume/lib/bundle-cut.mjs';
import { buildOrgSummary, ORG_FILES, FREE_CONTEXT_FILE }
from '../kb-update/lib/user-data.mjs';
const S = join(homedir(), 'okf-bakeoff-2026-08');
const ORG = join(homedir(), '.claude', 'ms-ai-architect', 'org');
const MAPPER = ['arkitektur', 'hendelser', 'leverandor', 'styring'];
// --- instrumentvalidering (C6): reproduser kjent-gode tall FOER egne tall brukes ---
// Amendement 2: A2b (3412) og A3 (5879) fra bake-off-dokumentet reproduserer IKKE
// fra de reddede artefaktene -- korpuset er verifisert uendret (55/55 sha256), og
// verken HEAD eller e4f1642 gir 3412. Triplet er byttet til de tre som faktisk
// reproduserer, og avviket rapporteres i stedet for aa brukes som referanse.
const KJENT_GODE = { a1: 31343, a2a: 2884, heleBundlen: 45580 };
function lesKorpus() {
const ut = [];
for (const mappe of MAPPER) {
for (const f of readdirSync(join(S, 'korpus', mappe)).sort()) {
const innhold = readFileSync(join(S, 'korpus', mappe, f), 'utf8');
ut.push({
id: /^id:\s*(\S+)/m.exec(innhold)[1],
mappe, fil: f, innhold,
flatnavn: `${mappe}__${f}`,
label: `${mappe}__${f.replace(/\.md$/, '')}`,
});
}
}
return ut;
}
function friKontekst(lengde, mal) {
const [fm, ...rest] = mal.split('## Fri kontekst');
let body = rest.join('## Fri kontekst').trim();
body = body.length >= lengde ? body.slice(0, lengde)
: body.padEnd(lengde, ` ${body}`).slice(0, lengde);
return `${fm}## Fri kontekst\n${body}\n`;
}
function validerInstrument(korpus) {
const kanoniske = {};
for (const f of ORG_FILES) kanoniske[f] = readFileSync(join(ORG, f), 'utf8');
const fri = friKontekst(1216, readFileSync(join(ORG, FREE_CONTEXT_FILE), 'utf8'));
const a1 = [...Object.values(kanoniske), fri, ...korpus.map((d) => d.innhold)].join('\n').length;
// A2a: dagens ikke-rekursive mekanisme -- bare de kanoniske org-filene naar fram.
const a2a = buildOrgSummary({ ...kanoniske, [FREE_CONTEXT_FILE]: fri }).length;
const bundleDir = join(S, 'bundle');
const heleBundlen = readdirSync(bundleDir)
.filter((f) => f.endsWith('.md'))
.reduce((n, f) => n + readFileSync(join(bundleDir, f), 'utf8').length, 0);
const maalt = { a1, a2a, heleBundlen };
const avvik = Object.keys(KJENT_GODE).filter((k) => maalt[k] !== KJENT_GODE[k]);
return { maalt, forventet: KJENT_GODE, avvik };
}
// --- kandidatbygging ---
const ID_FRA_LABEL = /__([A-ZÆØÅ]\d{2})-/;
function bundleKandidater(bundleDir, mode) {
const indexMd = readFileSync(join(bundleDir, 'index.md'), 'utf8');
const entries = parseIndex(indexMd);
let scanChars = 0;
const kandidater = entries.map(({ label, target }) => {
const body = readFileSync(join(bundleDir, target), 'utf8');
const k = { label, target, body, h2: [], id: (ID_FRA_LABEL.exec(label) || [])[1] };
if (mode === 'headscan') {
scanChars += body.split('\n').slice(0, HEAD_SCAN_LINES + 8).join('\n').length;
Object.assign(k, headScan(body));
}
return k;
});
return { kandidater, listeChars: indexMd.length, scanChars };
}
function reneKandidater(korpus, mode) {
// Ingen OKF-profil paastaar noe om denne katalogen, saa en listing er lovlig her
// (sjekkliste punkt 9 binder bare der en profil sier indeksen er forfattet).
const listeChars = korpus.map((d) => `${d.flatnavn}\n`).join('').length;
let scanChars = 0;
const kandidater = korpus.map((d) => {
const k = { label: d.label, body: d.innhold, h2: [], id: d.id };
if (mode === 'headscan') {
scanChars += d.innhold.split('\n').slice(0, HEAD_SCAN_LINES + 8).join('\n').length;
Object.assign(k, headScan(d.innhold));
}
return k;
});
return { kandidater, listeChars, scanChars };
}
// --- kjoer én arm over hele gullsettet ---
function kjoerArm({ navn, kandidater, listeChars, scanChars, mode, matcher, gull, k, budget }) {
const rader = [];
for (const g of gull) {
const qt = tokenize(g.spoersmaal);
const scoret = kandidater.map((c) => ({ ...c, score: scoreCandidate(qt, c, matcher) }));
const r = cutToBudget({
candidates: scoret, budget, k, applySupersession: mode === 'headscan',
});
const naadd = new Set(r.delivered.map((d) => d.id));
rader.push({
arm: navn, gull: g.id, klasse: g.klasse,
treff: g.fasit.every((id) => naadd.has(id)),
felle: g.felle.some((id) => naadd.has(id)),
levert: r.delivered.length,
unread: r.unread,
denominator: r.denominator,
superseded: r.superseded,
statusUnknown: r.statusUnknown,
agentChars: r.agentChars,
assemblyChars: listeChars + scanChars + r.agentChars,
});
}
return rader;
}
function oppsummer(rader) {
const klasser = ['enkeltdokument', 'to-dokument', 'stale-fersk', 'distraktor'];
const armer = [...new Set(rader.map((r) => r.arm))];
return armer.map((arm) => {
const r = rader.filter((x) => x.arm === arm);
const felleR = r.filter((x) => x.klasse === 'stale-fersk' || x.klasse === 'distraktor');
const snitt = (f) => Math.round(r.reduce((n, x) => n + f(x), 0) / r.length);
return {
arm,
treff: `${r.filter((x) => x.treff).length}/${r.length}`,
perKlasse: Object.fromEntries(klasser.map((kl) => {
const rk = r.filter((x) => x.klasse === kl);
return [kl, `${rk.filter((x) => x.treff).length}/${rk.length}`];
})),
felleEksponering: `${felleR.filter((x) => x.felle).length}/${felleR.length}`,
snittAgentTegn: snitt((x) => x.agentChars),
snittAssembleringTegn: snitt((x) => x.assemblyChars),
snittLevert: (r.reduce((n, x) => n + x.levert, 0) / r.length).toFixed(1),
snittUnread: `${(r.reduce((n, x) => n + x.unread, 0) / r.length).toFixed(1)}/${r[0].denominator}`,
};
});
}
function main(argv) {
if (!existsSync(S)) {
console.error(`Artefaktene mangler: ${S}. Se repoets STATE / bake-off-README.`);
process.exit(2);
}
const korpus = lesKorpus();
const gull = JSON.parse(readFileSync(join(S, 'gullsett.json'), 'utf8')).items;
const val = validerInstrument(korpus);
if (val.avvik.length) {
console.error('INSTRUMENTET REPRODUSERER IKKE DE KJENT-GODE TALLENE. Stanser.');
console.error(JSON.stringify(val, null, 2));
process.exit(1);
}
const bundleDir = join(S, 'bundle');
const bIdx = bundleKandidater(bundleDir, 'index');
const bHead = bundleKandidater(bundleDir, 'headscan');
const rHead = reneKandidater(korpus, 'headscan');
const armer = [
{ navn: 'B1 indeks (prereg)', ...bIdx, mode: 'index', matcher: tokensMatch },
{ navn: 'B2 hodeskann (prereg)', ...bHead, mode: 'headscan', matcher: tokensMatch },
{ navn: 'B3 rene filer (prereg)', ...rHead, mode: 'headscan', matcher: tokensMatch },
{ navn: 'B2+ hodeskann (post-hoc)', ...bHead, mode: 'headscan', matcher: tokensMatchCompound },
{ navn: 'B3+ rene filer (post-hoc)', ...rHead, mode: 'headscan', matcher: tokensMatchCompound },
];
const alle = [];
for (const budsjett of [{ k: Infinity, budget: AGENT_BUDGET, merk: 'budsjett 12000' },
{ k: 3, budget: AGENT_BUDGET, merk: 'k=3' }]) {
for (const a of armer) {
alle.push(...kjoerArm({ ...a, gull, k: budsjett.k, budget: budsjett.budget })
.map((r) => ({ ...r, regime: budsjett.merk })));
}
}
if (argv.includes('--json')) {
console.log(JSON.stringify({ instrument: val, rader: alle }, null, 2));
return;
}
console.log('INSTRUMENTVALIDERING (C6) — kjent-gode tall reprodusert:');
console.log(` A1 last-alt n=55 : ${val.maalt.a1} (forventet ${val.forventet.a1})`);
console.log(` A2a n=55 : ${val.maalt.a2a} (forventet ${val.forventet.a2a})`);
console.log(` Hele bundlen : ${val.maalt.heleBundlen} (forventet ${val.forventet.heleBundlen})`);
for (const regime of ['budsjett 12000', 'k=3']) {
console.log(`\n=== REGIME: ${regime} (nevner 40 gullspoersmaal, felle-nevner 20) ===`);
console.table(oppsummer(alle.filter((r) => r.regime === regime)));
}
}
if (import.meta.url === `file://${process.argv[1]}`) main(process.argv.slice(2));

144
scripts/okf-consume/cut.mjs Normal file
View file

@ -0,0 +1,144 @@
#!/usr/bin/env node
// cut.mjs — entry point for the DETERMINISTIC half of the OKF bundle
// consumption skill (contract C4: the script cuts, the agent judges).
//
// Usage:
// node scripts/okf-consume/cut.mjs --bundle <dir> --query "<question>" [--k N]
// [--budget N] [--mode index|headscan] [--json]
//
// The OKF envelope of each bundle document is parsed by llm-ingestion-okf
// itself (materialize.parse_frontmatter), invoked as a subprocess. Nothing in
// this repo re-implements it. Set OKF_PYTHON to point at that package's venv
// interpreter; without it the envelope is reported as unavailable rather than
// guessed at.
import { readFileSync, readdirSync, existsSync } from 'node:fs';
import { join } from 'node:path';
import { execFileSync } from 'node:child_process';
import {
tokenize, parseIndex, headScan, scoreCandidate, cutToBudget,
AGENT_BUDGET, HEAD_SCAN_LINES,
} from './lib/bundle-cut.mjs';
const OKF_PYTHON = process.env.OKF_PYTHON
|| '/Users/ktg/repos/llm-ingestion-okf/.venv/bin/python';
/**
* Parse the OKF envelope of every document in the bundle, using the producing
* package. Returns a map target -> envelope, or null when the interpreter is
* unavailable — never a hand-rolled fallback, because a fallback would be this
* repo owning a format it does not own.
*/
export function readEnvelopes(bundleDir, python = OKF_PYTHON) {
if (!existsSync(python)) return null;
const src = `
import json, sys
from pathlib import Path
from llm_ingestion_okf.materialize import parse_frontmatter
d = Path(sys.argv[1])
out = {}
for p in sorted(d.iterdir()):
if p.name == 'index.md' or p.suffix != '.md':
continue
try:
out[p.name] = parse_frontmatter(p)
except Exception as e:
out[p.name] = {'_error': type(e).__name__}
json.dump(out, sys.stdout)
`;
try {
return JSON.parse(execFileSync(python, ['-c', src, bundleDir], { encoding: 'utf8' }));
} catch {
return null;
}
}
/**
* Build the candidate set. In 'index' mode the ranker sees only what the index
* carries (labels); in 'headscan' mode it also sees the preserved frontmatter
* and H2 headings inside the first HEAD_SCAN_LINES body lines.
*
* assemblyChars counts what the SCRIPT read to get here. It is reported
* separately from what reaches the agent, because those are different costs and
* conflating them is how a cut looks cheaper than it is.
*/
export function buildCandidates({ bundleDir, mode = 'headscan', entries, readFile = readFileSync }) {
let assemblyChars = 0;
const candidates = [];
for (const { label, target } of entries) {
const path = join(bundleDir, target);
let text = '';
try { text = readFile(path, 'utf8'); } catch { continue; }
const cand = { label, target, body: text, h2: [] };
if (mode === 'headscan') {
const head = text.split('\n').slice(0, HEAD_SCAN_LINES + 8).join('\n');
assemblyChars += head.length;
Object.assign(cand, headScan(text));
}
candidates.push(cand);
}
return { candidates, assemblyChars };
}
export function runCut({ bundleDir, query, mode = 'headscan', k = Infinity, budget = AGENT_BUDGET }) {
const indexPath = join(bundleDir, 'index.md');
const indexMd = readFileSync(indexPath, 'utf8');
const entries = parseIndex(indexMd);
const { candidates, assemblyChars: scanChars } = buildCandidates({ bundleDir, mode, entries });
const queryTokens = tokenize(query);
for (const c of candidates) c.score = scoreCandidate(queryTokens, c);
const result = cutToBudget({
candidates, budget, k, applySupersession: mode === 'headscan',
});
const assemblyChars = indexMd.length + scanChars
+ result.delivered.reduce((n, d) => n + d.body.length, 0);
return { ...result, mode, queryTokens, indexChars: indexMd.length, scanChars, assemblyChars };
}
function main(argv) {
const arg = (name, fallback) => {
const i = argv.indexOf(`--${name}`);
return i === -1 ? fallback : argv[i + 1];
};
const bundleDir = arg('bundle');
const query = arg('query');
if (!bundleDir || !query) {
console.error('usage: cut.mjs --bundle <dir> --query "<question>" [--k N] [--budget N] [--mode index|headscan] [--json]');
process.exit(2);
}
const r = runCut({
bundleDir,
query,
mode: arg('mode', 'headscan'),
k: arg('k') ? Number(arg('k')) : Infinity,
budget: Number(arg('budget', AGENT_BUDGET)),
});
if (argv.includes('--json')) {
console.log(JSON.stringify({
...r, delivered: r.delivered.map((d) => ({ label: d.label, target: d.target, score: d.score })),
}, null, 2));
return;
}
// The receipt comes FIRST, so a reader cannot consume the cut without seeing
// what was held back.
console.log(`## Kutt-kvittering (OKF-bundle, modus: ${r.mode})`);
console.log(`- Levert: ${r.delivered.length} av ${r.denominator} dokumenter`);
console.log(`- \`[unread]\`: ${r.unread} av ${r.denominator} dokumenter i bundlen ble IKKE lest`);
if (r.superseded) console.log(`- Forkastet som \`status: erstattet\`: ${r.superseded}`);
if (r.statusUnknown) console.log(`- Uten \`status\`-felt (byggingen registrerte ingen status — ikke det samme som gjeldende): ${r.statusUnknown}`);
console.log(`- Agent-kontekst: ${r.agentChars} tegn av budsjett ${arg('budget', AGENT_BUDGET)}`);
console.log(`- Assemblering (hva skriptet leste): ${r.assemblyChars} tegn`);
console.log('');
for (const d of r.delivered) {
console.log(`### ${d.label} (score ${d.score})`);
console.log(d.body);
}
}
if (import.meta.url === `file://${process.argv[1]}`) main(process.argv.slice(2));

208
scripts/okf-consume/lib/bundle-cut.mjs Normal file
View file

@ -0,0 +1,208 @@
// bundle-cut.mjs — the DETERMINISTIC half of the OKF bundle consumption skill.
//
// Consumption contract (llm-ingestion-okf, docs/plan/okf-bundle-consumption-contract.md
// @ 01e4170) C4: "the script cuts, the agent judges". This module is the script.
// It reads, ranks and CUTS a bundle to a bounded context, and it always hands
// back a receipt for what it held back — a silent cut is C3's failure with
// extra steps.
//
// BOUNDARY against llm-ingestion-okf: nothing here writes, indexes, checks or
// retrieves in the sense that package owns. The OKF *envelope* is parsed by the
// package itself (materialize.parse_frontmatter, called as a subprocess by the
// driver); what this module parses out of the document BODY is the user's own
// corpus frontmatter, which OKF preserves verbatim but does not define.
//
// Pure functions only: no fs, no spawn, no clock. Everything is handed in.
// Weights locked in the pre-registration BEFORE any arm was run. Changing one
// invalidates the measurement, not just the code.
export const FIELD_WEIGHTS = Object.freeze({ tittel: 3, h2: 2, sjanger: 2, label: 1 });
// The bound the cut targets, in characters: 3x the plugin's existing 4000-char
// org-context budget (DEFAULT_MAX_TOTAL_LEN in kb-update/lib/user-data.mjs),
// because this cut delivers WHOLE documents rather than one-line field summaries.
export const AGENT_BUDGET = 12000;
// Same window A3-max used in the bake-off, so the arms stay comparable.
export const HEAD_SCAN_LINES = 16;
// Minimum shared prefix for two tokens to count as the same word. 4 is what
// Norwegian inflection needs (vedtak/vedtaket) without collapsing vei/veileder.
const MIN_PREFIX = 4;
// Standard Norwegian stop words (bokmaal + nynorsk). Only the 4-char-and-longer
// entries can ever fire, since shorter tokens are dropped on length anyway.
export const NORSKE_STOPPORD = new Set([
'alle', 'andre', 'annet', 'bare', 'begge', 'blir', 'blitt', 'bare', 'begge',
'denne', 'dere', 'deres', 'deim', 'deira', 'deires', 'dette', 'disse', 'ditt',
'dykk', 'dykkar', 'eller', 'elles', 'etter', 'fordi', 'hadde', 'hans', 'henne',
'hennar', 'hennes', 'here', 'hoss', 'hossen', 'hvem', 'hvilke', 'hvilken',
'hvis', 'hvor', 'hvordan', 'hvorfor', 'ikke', 'ikkje', 'ingen', 'ingi', 'inkje',
'inni', 'kine', 'korleis', 'korso', 'kunne', 'kvar', 'kvarhelst', 'kvifor',
'mange', 'medan', 'meget', 'mellom', 'mine', 'mitt', 'mykje', 'noen', 'noka',
'nokon', 'nokor', 'nokre', 'over', 'samme', 'siden', 'sidan', 'sine', 'sitt',
'skal', 'skulle', 'slik', 'somme', 'somt', 'sånn', 'uten', 'vart', 'varte',
'vere', 'verte', 'vore', 'vors', 'vort', 'ville', 'være', 'vært',
]);
/**
* NFC -> lowercase -> split on anything that is not a letter or digit ->
* drop tokens under MIN_PREFIX chars -> drop stop words.
*/
export function tokenize(text) {
if (!text) return [];
return String(text)
.normalize('NFC')
.toLowerCase()
.split(/[^\p{L}\p{N}]+/u)
.filter((t) => t.length >= MIN_PREFIX && !NORSKE_STOPPORD.has(t));
}
/** Two tokens are the same word if one prefixes the other by at least MIN_PREFIX. */
export function tokensMatch(a, b) {
if (!a || !b) return false;
const [short, long] = a.length <= b.length ? [a, b] : [b, a];
if (short.length < MIN_PREFIX) return false;
return long.startsWith(short);
}
// POST-HOC (pre-registration amendment 1): Norwegian is a compounding language,
// so "vektorindeks" and "vektorrepresentasjoner" are the same subject while
// neither prefixes the other. This matcher adds a shared-substring rule on top
// of the pre-registered prefix rule. It is a SUPERSET of tokensMatch and is
// labelled post-hoc everywhere it is reported.
const MIN_COMMON_SUBSTRING = 6;
export function tokensMatchCompound(a, b) {
if (tokensMatch(a, b)) return true;
if (!a || !b) return false;
for (let len = Math.min(a.length, b.length); len >= MIN_COMMON_SUBSTRING; len -= 1) {
for (let i = 0; i + len <= a.length; i += 1) {
if (b.includes(a.slice(i, i + len))) return true;
}
}
return false;
}
/**
* Read the bundle index. DEFAULT's link template is '- [{label}]({target})';
* the policy allows prose, so non-matching lines are skipped rather than an error.
*
* The index is read because DEFAULT sets entries_match_directory=False — the
* index is AUTHORED, so the directory is not its denominator and a listing is
* not a check (contract paragraph 2, checklist item 9).
*/
export function parseIndex(indexMd) {
const re = /^- \[([^\]]*)\]\(([^)]+)\)$/;
return String(indexMd ?? '')
.split('\n')
.map((line) => re.exec(line.trim()))
.filter(Boolean)
.map((m) => ({ label: m[1], target: m[2] }));
}
// The OKF envelope is recognised by a key only it writes. Recognising it by
// "starts with ---" would eat the user's own frontmatter on a non-bundle file.
const OKF_ENVELOPE_KEY = /^source_sha256:/m;
const PRESERVED_KEYS = ['id', 'tittel', 'sjanger', 'dato', 'status', 'superseded_by'];
/**
* Lift the user's preserved frontmatter and H2 headings out of the first
* HEAD_SCAN_LINES lines of the document BODY (the OKF envelope, if present,
* is stripped first and does not count against the window).
*
* A key that is absent is ABSENT — it is never defaulted. C5: absence of a
* conditionally-written field is a measurement, not the negation of what the
* field asserts.
*/
export function headScan(text, lines = HEAD_SCAN_LINES) {
let rest = String(text ?? '');
if (rest.startsWith('---\n')) {
const end = rest.indexOf('\n---', 3);
if (end !== -1) {
const block = rest.slice(0, end);
if (OKF_ENVELOPE_KEY.test(block)) rest = rest.slice(end + 4);
}
}
const window = rest.split('\n').slice(0, lines);
const out = { h2: [] };
for (const line of window) {
const h2 = /^##\s+(.+?)\s*$/.exec(line);
if (h2) { out.h2.push(h2[1]); continue; }
const kv = /^([a-zA-Z_]+):\s*(.+?)\s*$/.exec(line);
if (kv && PRESERVED_KEYS.includes(kv[1])) out[kv[1]] = kv[2];
}
return out;
}
/**
* Score = sum over DISTINCT query tokens of the highest field weight that token
* hit. Distinct, so a repeated word cannot inflate a document's rank.
*/
export function scoreCandidate(queryTokens, candidate, matcher = tokensMatch) {
const fields = [
[FIELD_WEIGHTS.tittel, tokenize(candidate.tittel)],
[FIELD_WEIGHTS.h2, (candidate.h2 ?? []).flatMap((h) => tokenize(h))],
[FIELD_WEIGHTS.sjanger, tokenize(candidate.sjanger)],
[FIELD_WEIGHTS.label, tokenize(candidate.label)],
];
let total = 0;
for (const q of new Set(queryTokens)) {
let best = 0;
for (const [weight, tokens] of fields) {
if (weight > best && tokens.some((t) => matcher(q, t))) best = weight;
}
total += best;
}
return total;
}
/**
* Take scored candidates and cut them to the budget, newest-first on ties.
*
* Returns the delivered documents AND the receipt: how many bundle documents
* were not delivered, out of how many there were. The denominator is the whole
* candidate set, never the shortlist — reporting against the shortlist is how a
* cut launders itself into looking like a complete read.
*/
export function cutToBudget({ candidates, budget = AGENT_BUDGET, k = Infinity, applySupersession = false }) {
const denominator = candidates.length;
let superseded = 0;
let statusUnknown = 0;
const eligible = candidates.filter((c) => {
if (applySupersession) {
if (c.status === 'erstattet') { superseded += 1; return false; }
if (c.status === undefined) statusUnknown += 1;
}
return true;
});
const ranked = eligible
.map((c, i) => ({ c, i }))
.filter(({ c }) => (c.score ?? 0) > 0)
.sort((a, b) => (b.c.score - a.c.score)
|| String(b.c.dato ?? '').localeCompare(String(a.c.dato ?? ''))
|| (a.i - b.i))
.map(({ c }) => c);
const delivered = [];
let agentChars = 0;
for (const c of ranked) {
if (delivered.length >= k) break;
const size = (c.body ?? '').length;
if (agentChars + size > budget) break;
delivered.push(c);
agentChars += size;
}
return {
delivered,
agentChars,
denominator,
unread: denominator - delivered.length,
superseded,
statusUnknown,
};
}

216
tests/kb-eval/test-okf-bundle-cut.test.mjs Normal file
View file

@ -0,0 +1,216 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
tokenize,
tokensMatch,
tokensMatchCompound,
parseIndex,
headScan,
scoreCandidate,
cutToBudget,
AGENT_BUDGET,
HEAD_SCAN_LINES,
FIELD_WEIGHTS,
} from '../../scripts/okf-consume/lib/bundle-cut.mjs';
// The cut is the whole contract. Everything the agent is allowed to judge comes
// through here, and everything it is NOT given has to leave a receipt behind
// ([unread], C1/C4). These tests are unforgiving about the receipt for that
// reason: a silent cut is C3's failure with extra steps.
// --- tokenisering: laast i forhaandsregistreringen paragraf 4 ---
test('tokenize lowercases, splits on non-letters and drops tokens under 4 chars', () => {
assert.deepEqual(tokenize('Kan vi LEGGE saksdokumenter i en vektorindeks?'),
['legge', 'saksdokumenter', 'vektorindeks']);
});
test('tokenize drops Norwegian stop words', () => {
// "dette", "eller" og "noen" er stoppord; "vedtak" er det ikke.
assert.deepEqual(tokenize('dette eller noen vedtak'), ['vedtak']);
});
test('tokenize keeps aeoeaa intact rather than mangling them', () => {
assert.deepEqual(tokenize('Sikkerhetsmålinger på løsningen'),
['sikkerhetsmålinger', 'løsningen']);
});
// --- prefiksmatching: norsk boeyning ---
test('tokensMatch accepts an inflected form via a 4-char common prefix', () => {
assert.equal(tokensMatch('vedtak', 'vedtaket'), true);
assert.equal(tokensMatch('dokumenter', 'dokument'), true);
});
test('tokensMatch refuses a 3-char overlap', () => {
assert.equal(tokensMatch('vei', 'veileder'), false);
assert.equal(tokensMatch('katt', 'kake'), false);
});
// --- indeks: forfattet, ikke avledet (DEFAULT: entries_match_directory=False) ---
test('parseIndex reads label and target from the DEFAULT link template', () => {
const idx = [
'- [arkitektur__A01-godkjente-ki-tjenester-v1](inbox-arkitektur-a01-godkjente-ki-tjenester-v1.md)',
'- [styring__S03-innkjop](inbox-styring-s03-innkjop.md)',
].join('\n');
assert.deepEqual(parseIndex(idx), [
{ label: 'arkitektur__A01-godkjente-ki-tjenester-v1', target: 'inbox-arkitektur-a01-godkjente-ki-tjenester-v1.md' },
{ label: 'styring__S03-innkjop', target: 'inbox-styring-s03-innkjop.md' },
]);
});
test('parseIndex ignores prose lines the DEFAULT policy allows', () => {
const idx = 'Noen innledende ord.\n\n- [a](inbox-a.md)\n';
assert.deepEqual(parseIndex(idx), [{ label: 'a', target: 'inbox-a.md' }]);
});
// --- hodeskann: bevart original-frontmatter ligger i KROPPEN ---
const bundleDoc = [
'---', 'type: note', 'title: arkitektur__A01-godkjente-ki-tjenester-v1',
'source_file: arkitektur__A01-godkjente-ki-tjenester-v1.md',
'source_sha256: deadbeef', 'ingested_at: 2026-08-26T12:00:00Z', 'generated: true', '---',
'', '---', 'id: A01', 'tittel: Katalog over godkjente KI-tjenester v1.0',
'sjanger: arkitekturprinsipp', 'dato: 2025-11-12', 'status: erstattet',
'superseded_by: A02', '---', '', '# Katalog over godkjente KI-tjenester v1.0',
'', '## Godkjente tjenester', 'Copilot for Microsoft 365.',
].join('\n');
test('headScan lifts the preserved original frontmatter out of the body', () => {
const h = headScan(bundleDoc);
assert.equal(h.tittel, 'Katalog over godkjente KI-tjenester v1.0');
assert.equal(h.sjanger, 'arkitekturprinsipp');
assert.equal(h.dato, '2025-11-12');
assert.equal(h.status, 'erstattet');
assert.equal(h.superseded_by, 'A02');
});
test('headScan reads only the first HEAD_SCAN_LINES lines', () => {
const padded = Array(HEAD_SCAN_LINES).fill('# filler').join('\n') + '\n' + bundleDoc;
assert.equal(headScan(padded).tittel, undefined);
});
test('headScan reports absent status as undefined, never as current (C5)', () => {
const noStatus = bundleDoc.split('\n').filter((l) => !l.startsWith('status:')).join('\n');
const h = headScan(noStatus);
assert.equal(h.status, undefined);
assert.equal('status' in h, false);
});
test('headScan collects H2 headings inside the scanned window', () => {
assert.deepEqual(headScan(bundleDoc).h2, ['Godkjente tjenester']);
});
// --- scoring: vekter laast foer kjoering ---
test('scoreCandidate weights tittel above sjanger above index label', () => {
const cand = { label: 'noe-annet', tittel: 'Vektorindeks for saksdokumenter', sjanger: 'annet', h2: [] };
assert.equal(scoreCandidate(['vektorindeks'], cand), FIELD_WEIGHTS.tittel);
const viaSjanger = { label: 'noe-annet', tittel: 'Uten treff', sjanger: 'vektorindeks', h2: [] };
assert.equal(scoreCandidate(['vektorindeks'], viaSjanger), FIELD_WEIGHTS.sjanger);
const viaLabel = { label: 'vektorindeks-notat', tittel: 'Uten treff', sjanger: 'annet', h2: [] };
assert.equal(scoreCandidate(['vektorindeks'], viaLabel), FIELD_WEIGHTS.label);
});
test('scoreCandidate counts a repeated query token once', () => {
const cand = { label: 'vektorindeks', tittel: 'Vektorindeks', sjanger: 'vektorindeks', h2: ['Vektorindeks'] };
assert.equal(scoreCandidate(['vektorindeks', 'vektorindeks'], cand), FIELD_WEIGHTS.tittel);
});
test('scoreCandidate takes the highest-weighted field a token hit', () => {
const cand = { label: 'vektorindeks', tittel: 'Vektorindeks', sjanger: 'annet', h2: [] };
assert.equal(scoreCandidate(['vektorindeks'], cand), FIELD_WEIGHTS.tittel);
});
// --- kuttet: budsjett, supersesjon, og kvitteringen ---
const mkCand = (id, score, chars, extra = {}) => ({
id, label: id, tittel: id, sjanger: 'x', h2: [], dato: '2026-01-01',
body: 'x'.repeat(chars), score, ...extra,
});
test('cutToBudget delivers highest score first and stops before overflowing', () => {
const cands = [mkCand('a', 1, 5000), mkCand('b', 9, 5000), mkCand('c', 5, 5000)];
const r = cutToBudget({ candidates: cands, budget: 11000 });
assert.deepEqual(r.delivered.map((d) => d.id), ['b', 'c']);
assert.ok(r.agentChars <= 11000);
});
test('cutToBudget reports unread against the full denominator, not the shortlist', () => {
const cands = [mkCand('a', 1, 5000), mkCand('b', 9, 5000), mkCand('c', 5, 5000)];
const r = cutToBudget({ candidates: cands, budget: 11000 });
assert.equal(r.denominator, 3);
assert.equal(r.unread, 1);
});
test('cutToBudget drops superseded documents mechanically', () => {
const cands = [mkCand('gammel', 9, 100, { status: 'erstattet' }), mkCand('fersk', 1, 100)];
const r = cutToBudget({ candidates: cands, budget: AGENT_BUDGET, applySupersession: true });
assert.deepEqual(r.delivered.map((d) => d.id), ['fersk']);
assert.equal(r.superseded, 1);
});
test('cutToBudget keeps a document whose status is absent (C5: absence is not a fact)', () => {
const cands = [mkCand('ukjent', 5, 100)];
const r = cutToBudget({ candidates: cands, budget: AGENT_BUDGET, applySupersession: true });
assert.deepEqual(r.delivered.map((d) => d.id), ['ukjent']);
assert.equal(r.statusUnknown, 1);
});
test('cutToBudget never delivers a zero-scoring candidate', () => {
const r = cutToBudget({ candidates: [mkCand('a', 0, 100)], budget: AGENT_BUDGET });
assert.deepEqual(r.delivered, []);
assert.equal(r.unread, 1);
});
test('cutToBudget breaks score ties on dato descending, then index order', () => {
const cands = [
mkCand('eldst', 4, 100, { dato: '2024-01-01' }),
mkCand('nyest', 4, 100, { dato: '2026-05-05' }),
mkCand('midt', 4, 100, { dato: '2025-01-01' }),
];
const r = cutToBudget({ candidates: cands, budget: AGENT_BUDGET });
assert.deepEqual(r.delivered.map((d) => d.id), ['nyest', 'midt', 'eldst']);
});
test('cutToBudget honours an explicit k cap for comparability with the bake-off', () => {
const cands = [mkCand('a', 9, 100), mkCand('b', 8, 100), mkCand('c', 7, 100), mkCand('d', 6, 100)];
const r = cutToBudget({ candidates: cands, budget: AGENT_BUDGET, k: 3 });
assert.equal(r.delivered.length, 3);
assert.equal(r.unread, 1);
});
test('AGENT_BUDGET is the pre-registered 12000 chars', () => {
assert.equal(AGENT_BUDGET, 12000);
assert.equal(HEAD_SCAN_LINES, 16);
});
// --- POST-HOC (amendement 1): sammensetningsmatching for norsk ---
test('tokensMatchCompound accepts a shared compound head of 6 chars', () => {
// vektorindeks / vektorrepresentasjoner deler "vektor" (6).
assert.equal(tokensMatchCompound('vektorindeks', 'vektorrepresentasjoner'), true);
});
test('tokensMatchCompound accepts a shared compound tail', () => {
// saksdokumenter / kildedokumentet deler "dokument" (8).
assert.equal(tokensMatchCompound('saksdokumenter', 'kildedokumentet'), true);
});
test('tokensMatchCompound still refuses a 5-char overlap', () => {
assert.equal(tokensMatchCompound('kontor', 'kontant'), false);
});
test('tokensMatchCompound remains a superset of the pre-registered rule', () => {
assert.equal(tokensMatchCompound('vedtak', 'vedtaket'), true);
assert.equal(tokensMatchCompound('vei', 'veileder'), false);
});
test('scoreCandidate can be run with an injected matcher without changing the default', () => {
const cand = { label: 'x', tittel: 'Lagring av vektorrepresentasjoner', sjanger: 'y', h2: [] };
assert.equal(scoreCandidate(['vektorindeks'], cand), 0);
assert.equal(scoreCandidate(['vektorindeks'], cand, tokensMatchCompound), FIELD_WEIGHTS.tittel);
});