diff --git a/lib/convert/index.mjs b/lib/convert/index.mjs new file mode 100644 index 0000000..873daaa --- /dev/null +++ b/lib/convert/index.mjs @@ -0,0 +1,132 @@ +// convert/index.mjs +// Step 11 (SC format): konverterings-adaptere for innboks-ingestion. +// Dispatch paa extension: .txt/.md -> node:-builtins (les direkte); +// .docx -> mammoth(->HTML)->turndown; .eml -> postal-mime (kun body -- +// vedlegg = dokumentert v1-non-goal); .pdf -> unpdf tekst (flat, ingen +// headings -- dokumentert v1-caveat M5). Ukjent extension -> skip + norsk +// notice (returner null), IKKE feil -- en enkelt ukjent fil skal aldri felle +// hele kjoeringen. +// +// Sikkerhet (innboks-dok = fiendtlig): +// - INGEN nettverk: all input leses fra disk og mates som buffer/streng. +// - CVE-2024-4367: unpdf kalles med eksplisitt { isEvalSupported: false } +// (default i unpdf 1.6.2, A0-verifisert -- settes likevel, belte+seler). +// - Eksterne lenker NOEYTRALISERES ved konvertering (B3/F2): lenketekst +// bevares, scheme-/ikke-bundle-maal droppes -- i ALLE lenkeformer +// (inline, referanse-definisjon, autolink, raa HTML-anker, bilde). Kun +// trygge bundle-root-relative .md-lenker (isSafeBundleLink, den delte +// allow-listen) overlever. Legitim konvertert output kan dermed aldri +// felles av strict-gaten pga. medbrakte eksterne lenker. +// - M5/F4: turndown pinnes til { headingStyle: 'atx' } -- default er setext, +// som heading-splitten (innboks-split) aldri ser. +// +// npm-deps (mammoth/turndown/postal-mime/unpdf) er dokumentert prerequisite +// (exact-pinnet i package.json, Step 10) -- mangler de, kastes en klar norsk +// installasjonshint (samme moenster som scripts/export-pdf.py), aldri traceback. + +import { readFileSync } from 'node:fs'; +import path from 'node:path'; + +import { isSafeBundleLink } from '../okf-links.mjs'; + +const INSTALL_HINT = 'kjoer «npm install --ignore-scripts» i plugin-rota (krever Node >= 22)'; + +// Lazy-import av en konverterings-motor; mangler den -> klar norsk hint. +async function loadEngine(name) { + try { + return await import(name); + } catch (e) { + if (e && (e.code === 'ERR_MODULE_NOT_FOUND' || e.code === 'MODULE_NOT_FOUND')) { + throw new Error(`innboks: mangler npm-avhengighet «${name}» -- ${INSTALL_HINT}`); + } + throw e; + } +} + +// M5/F4-pinning: atx ('#') er OBLIGATORISK for at heading-splitten skal virke. +async function newTurndown() { + const TurndownService = (await loadEngine('turndown')).default; + return new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' }); +} + +// Noeytraliser alle lenkeformer med ikke-bundle-maal; behold lenketeksten. +// Rekkefoelgen er semantisk: HTML-anker foer autolink (begge bruker <...>), +// ref-definisjoner foer ref-bruk-kollaps, bilder foer inline (![..](..) baerer +// samme hale som [..](..)). +export function neutralizeExternalLinks(markdown) { + let out = String(markdown); + // 1. Raa HTML-anker -> indre tekst (file://-href o.l. droppes med taggen). + out = out.replace(/]*>([\s\S]*?)<\/a>/gi, '$1'); + out = out.replace(/<\/?a\b[^>]*>/gi, ''); + // 2. Autolink -> ren tekst (URL mister lenke-formen). + out = out.replace(/<([a-z][a-z0-9+.-]*:[^<>\s]*)>/gi, '$1'); + // 3. Referanse-definisjoner droppes linjevis; ref-bruk kollapses til tekst. + out = out.replace(/^[ \t]{0,3}\[[^\]]+\]:[ \t]*\S.*$/gm, ''); + out = out.replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1'); + // 4. Bilder -> alt-tekst (EchoLeak-stil exfil-piksel har ingen plass i bundlen). + out = out.replace(/!\[([^\]]*)\]\(([^)]*)\)/g, '$1'); + // 5. Inline-lenker: KUN trygge bundle-root-relative .md-maal beholdes. + out = out.replace(/\[([^\]]*)\]\(([^)]+)\)/g, (whole, text, target) => { + const dest = target.trim().split(/\s+/)[0]; + return isSafeBundleLink(dest) ? whole : text; + }); + return out; +} + +async function docxToMarkdown(filePath) { + const mammothMod = await loadEngine('mammoth'); + const mammoth = mammothMod.default ?? mammothMod; + const { value: html } = await mammoth.convertToHtml({ buffer: readFileSync(filePath) }); + return (await newTurndown()).turndown(html); +} + +async function emlToMarkdown(filePath) { + const PostalMime = (await loadEngine('postal-mime')).default; + const parsed = await new PostalMime().parse(readFileSync(filePath, 'utf8')); + const parts = []; + if (parsed.subject) parts.push(`# ${parsed.subject}`); + if (parsed.text) { + parts.push(parsed.text.trim()); + } else if (parsed.html) { + parts.push((await newTurndown()).turndown(parsed.html)); + } + return parts.join('\n\n'); +} + +async function pdfToMarkdown(filePath) { + const { getDocumentProxy, extractText } = await loadEngine('unpdf'); + const data = new Uint8Array(readFileSync(filePath)); + const proxy = await getDocumentProxy(data, { isEvalSupported: false }); + const { text } = await extractText(proxy, { mergePages: true }); + return text; +} + +// filsti -> markdown-streng (neoytralisert, garantert trailing newline), +// eller null for ukjent extension (skip + norsk notice via onNotice). +export async function convert( + filePath, + { onNotice = (msg) => process.stderr.write(`${msg}\n`) } = {}, +) { + const ext = path.extname(filePath).toLowerCase(); + let markdown; + switch (ext) { + case '.md': + case '.txt': + markdown = readFileSync(filePath, 'utf8'); + break; + case '.docx': + markdown = await docxToMarkdown(filePath); + break; + case '.eml': + markdown = await emlToMarkdown(filePath); + break; + case '.pdf': + markdown = await pdfToMarkdown(filePath); + break; + default: + onNotice(`innboks: hopper over ukjent filtype «${ext || '(ingen)'}»: ${path.basename(filePath)}`); + return null; + } + const neutralized = neutralizeExternalLinks(markdown); + return neutralized.endsWith('\n') ? neutralized : `${neutralized}\n`; +} diff --git a/tests/fixtures/inbox-sample/dok.docx b/tests/fixtures/inbox-sample/dok.docx new file mode 100644 index 0000000..85384b7 Binary files /dev/null and b/tests/fixtures/inbox-sample/dok.docx differ diff --git a/tests/fixtures/inbox-sample/dok.eml b/tests/fixtures/inbox-sample/dok.eml new file mode 100644 index 0000000..17d89a6 --- /dev/null +++ b/tests/fixtures/inbox-sample/dok.eml @@ -0,0 +1,11 @@ +From: styringsstab@etat.no +To: okr-team@etat.no +Subject: Statusoppdatering T2 +Date: Mon, 18 May 2026 09:00:00 +0200 +MIME-Version: 1.0 +Content-Type: text/plain; charset=utf-8 + +Fremdrift paa maaltallene for andre tertial. + +KR1 ligger paa 0.6 etter maai. KR2 er i rute. Vi maa diskutere +ambisjonsnivaaet for KR3 paa neste check-in. diff --git a/tests/fixtures/inbox-sample/dok.pdf b/tests/fixtures/inbox-sample/dok.pdf new file mode 100644 index 0000000..c7a76c0 --- /dev/null +++ b/tests/fixtures/inbox-sample/dok.pdf @@ -0,0 +1,10 @@ +%PDF-1.4 +1 0 obj<>endobj +2 0 obj<>endobj +3 0 obj<>>>>>endobj +4 0 obj<>stream +BT /F1 12 Tf 72 720 Td (Tildelingsbrev 2026 for etaten) Tj ET +endstream +endobj +5 0 obj<>endobj +trailer<> \ No newline at end of file diff --git a/tests/innboks-convert.test.mjs b/tests/innboks-convert.test.mjs new file mode 100644 index 0000000..6bdf2cb --- /dev/null +++ b/tests/innboks-convert.test.mjs @@ -0,0 +1,140 @@ +// innboks-convert.test.mjs +// Step 11 (SC format): konverterings-adaptere txt/md/docx/eml/pdf -> markdown. +// Verifiserer: +// - .txt/.md -> markdown alltid (zero-dep-sti, node:-builtins) +// - eksterne lenker NOEYTRALISERES ved konvertering (inline/ref-def/autolink/ +// HTML-anker/bilde): lenketeksten bevares, scheme-maalet droppes -- slik at +// strict-gaten (B3/F2) aldri felles av legitim konvertert output +// - .docx/.eml -> heading-STRUKTUR bevart ('# '-linjer, atx -- IKKE byte-snapshot); +// .pdf -> tekst ekstrahert (flat, dokumentert v1-caveat M5) +// - convert-twice-identical: samme binaerfil konvertert 2x gir byte-identisk +// markdown (konverterings-determinismen idempotensen hviler paa) +// - ukjent extension (.xlsx) -> null + norsk notice, INGEN throw +// Binaer-adapterne kjoerer conditional { skip: !engines } (plugin kan vaere +// installert uten node_modules -- npm-deps er dokumentert prerequisite). +// Moenster: node:test conditional skip; fixtures under tests/fixtures/inbox-sample/. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { convert } from '../lib/convert/index.mjs'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); +const SAMPLE = join(ROOT, 'tests', 'fixtures', 'inbox-sample'); + +// Er binaer-konverterings-motorene installert? (npm-deps = dokumentert prerequisite.) +async function enginesAvailable() { + try { + await import('mammoth'); + await import('turndown'); + await import('postal-mime'); + await import('unpdf'); + return true; + } catch { + return false; + } +} +const engines = await enginesAvailable(); + +// Temp-katalog for haandlagde input-filer. Ryddes alltid. +function withTmp(fn) { + const dir = mkdtempSync(join(tmpdir(), 'innboks-convert-')); + try { + return fn(dir); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +test('.txt konverteres alltid (zero-dep): innhold bevart som markdown', async () => { + const md = await convert(join(SAMPLE, 'dok-a.txt')); + assert.equal(typeof md, 'string'); + assert.ok(md.length > 0); +}); + +test('.md passthrough: markdown-struktur bevart', async () => { + await withTmp(async (dir) => { + const p = join(dir, 'notat.md'); + writeFileSync(p, '# Tittel\n\nAvsnitt.\n\n## Under\n\nMer.\n'); + const md = await convert(p); + assert.match(md, /^# Tittel$/m); + assert.match(md, /^## Under$/m); + }); +}); + +test('eksterne lenker noeytraliseres: inline/ref-def/autolink/HTML/bilde', async () => { + await withTmp(async (dir) => { + const p = join(dir, 'fiendtlig.txt'); + writeFileSync( + p, + '# Notat\n\n' + + 'Se [rapporten](https://evil.example/exfil) og [vedlegget][r].\n\n' + + '[r]: https://evil.example/ref\n\n' + + '\n\n' + + 'passord\n\n' + + '![skjermbilde](https://evil.example/pixel.png)\n\n' + + 'Intern relasjon: [Notat](/dokumenter/notat.md) beholdes.\n', + ); + const md = await convert(p); + // Lenketekst bevart: + assert.match(md, /rapporten/); + assert.match(md, /vedlegget/); + assert.match(md, /passord/); + assert.match(md, /skjermbilde/); + // Ingen lenke-FORM med eksterne maal igjen (inline, ref-def, autolink, HTML): + assert.doesNotMatch(md, /\]\(https?:/); + assert.doesNotMatch(md, /^\s{0,3}\[[^\]]+\]:/m); + assert.doesNotMatch(md, / markdown med atx-heading-struktur (ikke byte-snapshot)', { skip: !engines }, async () => { + const md = await convert(join(SAMPLE, 'dok.docx')); + assert.equal(typeof md, 'string'); + // M5/F4: {headingStyle:'atx'} er pinnet -> '#'-headings, aldri setext. + assert.match(md, /^# /m); + assert.match(md, /^## /m); + assert.doesNotMatch(md, /^=+$/m); +}); + +test('.eml -> markdown: subject som heading + body-tekst (kun body, vedlegg = non-goal)', { skip: !engines }, async () => { + const md = await convert(join(SAMPLE, 'dok.eml')); + assert.match(md, /^# Statusoppdatering T2/m); + assert.match(md, /Fremdrift paa maaltallene/); +}); + +test('.pdf -> tekst ekstrahert (flat markdown, isEvalSupported:false-sti)', { skip: !engines }, async () => { + const md = await convert(join(SAMPLE, 'dok.pdf')); + assert.equal(typeof md, 'string'); + assert.match(md, /Tildelingsbrev 2026/); +}); + +test('convert-twice-identical: docx og pdf gir byte-identisk markdown', { skip: !engines }, async () => { + for (const f of ['dok.docx', 'dok.pdf']) { + const first = await convert(join(SAMPLE, f)); + const second = await convert(join(SAMPLE, f)); + assert.equal(first, second, `ikke-deterministisk konvertering: ${f}`); + } +}); + +test('ukjent extension (.xlsx) -> null + norsk notice, ingen throw', async () => { + await withTmp(async (dir) => { + const p = join(dir, 'regneark.xlsx'); + writeFileSync(p, 'ikke egentlig xlsx'); + const notices = []; + const md = await convert(p, { onNotice: (msg) => notices.push(msg) }); + assert.equal(md, null); + assert.equal(notices.length, 1); + assert.match(notices[0], /hopper over/); + assert.match(notices[0], /regneark\.xlsx/); + }); +});