import { test } from 'node:test'; import assert from 'node:assert/strict'; import { tokenize, tokensMatch, tokensMatchCompound, parseIndex, headScan, scoreCandidate, cutToBudget, AGENT_BUDGET, HEAD_SCAN_LINES, FIELD_WEIGHTS, } from '../../scripts/okf-consume/lib/bundle-cut.mjs'; // The cut is the whole contract. Everything the agent is allowed to judge comes // through here, and everything it is NOT given has to leave a receipt behind // ([unread], C1/C4). These tests are unforgiving about the receipt for that // reason: a silent cut is C3's failure with extra steps. // --- tokenisering: laast i forhaandsregistreringen paragraf 4 --- test('tokenize lowercases, splits on non-letters and drops tokens under 4 chars', () => { assert.deepEqual(tokenize('Kan vi LEGGE saksdokumenter i en vektorindeks?'), ['legge', 'saksdokumenter', 'vektorindeks']); }); test('tokenize drops Norwegian stop words', () => { // "dette", "eller" og "noen" er stoppord; "vedtak" er det ikke. assert.deepEqual(tokenize('dette eller noen vedtak'), ['vedtak']); }); test('tokenize keeps aeoeaa intact rather than mangling them', () => { assert.deepEqual(tokenize('Sikkerhetsmålinger på løsningen'), ['sikkerhetsmålinger', 'løsningen']); }); // --- prefiksmatching: norsk boeyning --- test('tokensMatch accepts an inflected form via a 4-char common prefix', () => { assert.equal(tokensMatch('vedtak', 'vedtaket'), true); assert.equal(tokensMatch('dokumenter', 'dokument'), true); }); test('tokensMatch refuses a 3-char overlap', () => { assert.equal(tokensMatch('vei', 'veileder'), false); assert.equal(tokensMatch('katt', 'kake'), false); }); // --- indeks: forfattet, ikke avledet (DEFAULT: entries_match_directory=False) --- test('parseIndex reads label and target from the DEFAULT link template', () => { const idx = [ '- [arkitektur__A01-godkjente-ki-tjenester-v1](inbox-arkitektur-a01-godkjente-ki-tjenester-v1.md)', '- [styring__S03-innkjop](inbox-styring-s03-innkjop.md)', ].join('\n'); assert.deepEqual(parseIndex(idx), [ { label: 'arkitektur__A01-godkjente-ki-tjenester-v1', target: 'inbox-arkitektur-a01-godkjente-ki-tjenester-v1.md' }, { label: 'styring__S03-innkjop', target: 'inbox-styring-s03-innkjop.md' }, ]); }); test('parseIndex ignores prose lines the DEFAULT policy allows', () => { const idx = 'Noen innledende ord.\n\n- [a](inbox-a.md)\n'; assert.deepEqual(parseIndex(idx), [{ label: 'a', target: 'inbox-a.md' }]); }); // --- hodeskann: bevart original-frontmatter ligger i KROPPEN --- const bundleDoc = [ '---', 'type: note', 'title: arkitektur__A01-godkjente-ki-tjenester-v1', 'source_file: arkitektur__A01-godkjente-ki-tjenester-v1.md', 'source_sha256: deadbeef', 'ingested_at: 2026-08-26T12:00:00Z', 'generated: true', '---', '', '---', 'id: A01', 'tittel: Katalog over godkjente KI-tjenester v1.0', 'sjanger: arkitekturprinsipp', 'dato: 2025-11-12', 'status: erstattet', 'superseded_by: A02', '---', '', '# Katalog over godkjente KI-tjenester v1.0', '', '## Godkjente tjenester', 'Copilot for Microsoft 365.', ].join('\n'); test('headScan lifts the preserved original frontmatter out of the body', () => { const h = headScan(bundleDoc); assert.equal(h.tittel, 'Katalog over godkjente KI-tjenester v1.0'); assert.equal(h.sjanger, 'arkitekturprinsipp'); assert.equal(h.dato, '2025-11-12'); assert.equal(h.status, 'erstattet'); assert.equal(h.superseded_by, 'A02'); }); test('headScan reads only the first HEAD_SCAN_LINES lines', () => { const padded = Array(HEAD_SCAN_LINES).fill('# filler').join('\n') + '\n' + bundleDoc; assert.equal(headScan(padded).tittel, undefined); }); test('headScan reports absent status as undefined, never as current (C5)', () => { const noStatus = bundleDoc.split('\n').filter((l) => !l.startsWith('status:')).join('\n'); const h = headScan(noStatus); assert.equal(h.status, undefined); assert.equal('status' in h, false); }); test('headScan collects H2 headings inside the scanned window', () => { assert.deepEqual(headScan(bundleDoc).h2, ['Godkjente tjenester']); }); // --- scoring: vekter laast foer kjoering --- test('scoreCandidate weights tittel above sjanger above index label', () => { const cand = { label: 'noe-annet', tittel: 'Vektorindeks for saksdokumenter', sjanger: 'annet', h2: [] }; assert.equal(scoreCandidate(['vektorindeks'], cand), FIELD_WEIGHTS.tittel); const viaSjanger = { label: 'noe-annet', tittel: 'Uten treff', sjanger: 'vektorindeks', h2: [] }; assert.equal(scoreCandidate(['vektorindeks'], viaSjanger), FIELD_WEIGHTS.sjanger); const viaLabel = { label: 'vektorindeks-notat', tittel: 'Uten treff', sjanger: 'annet', h2: [] }; assert.equal(scoreCandidate(['vektorindeks'], viaLabel), FIELD_WEIGHTS.label); }); test('scoreCandidate counts a repeated query token once', () => { const cand = { label: 'vektorindeks', tittel: 'Vektorindeks', sjanger: 'vektorindeks', h2: ['Vektorindeks'] }; assert.equal(scoreCandidate(['vektorindeks', 'vektorindeks'], cand), FIELD_WEIGHTS.tittel); }); test('scoreCandidate takes the highest-weighted field a token hit', () => { const cand = { label: 'vektorindeks', tittel: 'Vektorindeks', sjanger: 'annet', h2: [] }; assert.equal(scoreCandidate(['vektorindeks'], cand), FIELD_WEIGHTS.tittel); }); // --- kuttet: budsjett, supersesjon, og kvitteringen --- const mkCand = (id, score, chars, extra = {}) => ({ id, label: id, tittel: id, sjanger: 'x', h2: [], dato: '2026-01-01', body: 'x'.repeat(chars), score, ...extra, }); test('cutToBudget delivers highest score first and stops before overflowing', () => { const cands = [mkCand('a', 1, 5000), mkCand('b', 9, 5000), mkCand('c', 5, 5000)]; const r = cutToBudget({ candidates: cands, budget: 11000 }); assert.deepEqual(r.delivered.map((d) => d.id), ['b', 'c']); assert.ok(r.agentChars <= 11000); }); test('cutToBudget reports unread against the full denominator, not the shortlist', () => { const cands = [mkCand('a', 1, 5000), mkCand('b', 9, 5000), mkCand('c', 5, 5000)]; const r = cutToBudget({ candidates: cands, budget: 11000 }); assert.equal(r.denominator, 3); assert.equal(r.unread, 1); }); test('cutToBudget drops superseded documents mechanically', () => { const cands = [mkCand('gammel', 9, 100, { status: 'erstattet' }), mkCand('fersk', 1, 100)]; const r = cutToBudget({ candidates: cands, budget: AGENT_BUDGET, applySupersession: true }); assert.deepEqual(r.delivered.map((d) => d.id), ['fersk']); assert.equal(r.superseded, 1); }); test('cutToBudget keeps a document whose status is absent (C5: absence is not a fact)', () => { const cands = [mkCand('ukjent', 5, 100)]; const r = cutToBudget({ candidates: cands, budget: AGENT_BUDGET, applySupersession: true }); assert.deepEqual(r.delivered.map((d) => d.id), ['ukjent']); assert.equal(r.statusUnknown, 1); }); test('cutToBudget never delivers a zero-scoring candidate', () => { const r = cutToBudget({ candidates: [mkCand('a', 0, 100)], budget: AGENT_BUDGET }); assert.deepEqual(r.delivered, []); assert.equal(r.unread, 1); }); test('cutToBudget breaks score ties on dato descending, then index order', () => { const cands = [ mkCand('eldst', 4, 100, { dato: '2024-01-01' }), mkCand('nyest', 4, 100, { dato: '2026-05-05' }), mkCand('midt', 4, 100, { dato: '2025-01-01' }), ]; const r = cutToBudget({ candidates: cands, budget: AGENT_BUDGET }); assert.deepEqual(r.delivered.map((d) => d.id), ['nyest', 'midt', 'eldst']); }); test('cutToBudget honours an explicit k cap for comparability with the bake-off', () => { const cands = [mkCand('a', 9, 100), mkCand('b', 8, 100), mkCand('c', 7, 100), mkCand('d', 6, 100)]; const r = cutToBudget({ candidates: cands, budget: AGENT_BUDGET, k: 3 }); assert.equal(r.delivered.length, 3); assert.equal(r.unread, 1); }); test('AGENT_BUDGET is the pre-registered 12000 chars', () => { assert.equal(AGENT_BUDGET, 12000); assert.equal(HEAD_SCAN_LINES, 16); }); // --- POST-HOC (amendement 1): sammensetningsmatching for norsk --- test('tokensMatchCompound accepts a shared compound head of 6 chars', () => { // vektorindeks / vektorrepresentasjoner deler "vektor" (6). assert.equal(tokensMatchCompound('vektorindeks', 'vektorrepresentasjoner'), true); }); test('tokensMatchCompound accepts a shared compound tail', () => { // saksdokumenter / kildedokumentet deler "dokument" (8). assert.equal(tokensMatchCompound('saksdokumenter', 'kildedokumentet'), true); }); test('tokensMatchCompound still refuses a 5-char overlap', () => { assert.equal(tokensMatchCompound('kontor', 'kontant'), false); }); test('tokensMatchCompound remains a superset of the pre-registered rule', () => { assert.equal(tokensMatchCompound('vedtak', 'vedtaket'), true); assert.equal(tokensMatchCompound('vei', 'veileder'), false); }); test('scoreCandidate can be run with an injected matcher without changing the default', () => { const cand = { label: 'x', tittel: 'Lagring av vektorrepresentasjoner', sjanger: 'y', h2: [] }; assert.equal(scoreCandidate(['vektorindeks'], cand), 0); assert.equal(scoreCandidate(['vektorindeks'], cand, tokensMatchCompound), FIELD_WEIGHTS.tittel); });