okr/tests/innboks-ingest.test.mjs

// innboks-ingest.test.mjs
// Step 12 (A2): end-to-end-test av pipeline-orkestratoren scripts/innboks-ingest.mjs.
// Kjoeres i mkdtempSync UTENFOR .claude/ med innboksen NESTET under bundle-rota
// (<tmp>/innboks inni <tmp>) saa walk-skippet ekserseres end-to-end. Verifiserer
// (plan.md Step 12, verifies 1-8):
//   (1) idempotens: 2 kjoeringer med ULIK veggklokke -> sha256-manifest deepEqual
//       (by-construction-determinisme via original-mtime, ikke cache)
//   (2) konformitet: okf-check subprosess BAADE default OG --strict-ingest exit 0
//   (3) index-integritet: generateIndexes 2x byte-identisk OG rot-index lister
//       ikke innboks/
//   (4) original-bevaring: drop-zone-originalene er byte-uendret (sha256)
//   (5) >= 1 relasjon: kryss-dokument tittel-omtale (dok-a -> dok-b) emitteres
//       som bundle-root-relativ .md-lenke (SC-regex)
//   (6) .txt-only full-pipeline kjoerer groent zero-dep (ingen npm-deps)
//   (7) strict-scoping (B2, laast A0): pre-eksisterende kuratert fil med ekstern
//       lenke + out-of-vocab-type feller IKKE kjoeringen; kuratert fil uroert
//   (8) per-dokument-staging/gate (laast A0): multi-doc-drop der ett dok feiler
//       gaten -> KUN det dokumentet discardes (ingen spor i tre/index), det
//       andre bestaar
// Binaer-originaler (.docx/.pdf) testes conditional { skip: !engines } -- B1-
// aksept: en ren kjoering med ikke-.md-original passerer strict.
// Moenster: tests/okf-check.test.mjs (subprosess-exit) + innboks-write.test.mjs.

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import {
  mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync, copyFileSync,
  existsSync, rmSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import { join, dirname, relative } from 'node:path';
import { fileURLToPath } from 'node:url';

import { ingestInbox } from '../scripts/innboks-ingest.mjs';

const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const CHECK = join(ROOT, 'scripts', 'okf-check.mjs');
const SAMPLE = join(ROOT, 'tests', 'fixtures', 'inbox-sample');

// Er binaer-konverterings-motorene installert? (npm-deps = dokumentert prerequisite.)
async function enginesAvailable() {
  try {
    await import('mammoth');
    await import('turndown');
    await import('unpdf');
    return true;
  } catch {
    return false;
  }
}
const engines = await enginesAvailable();

// Temp-bundle: bundle-rot = <tmp>, drop-zone = <tmp>/innboks (nestet, som prod).
async function withBundle(fn) {
  const bundleRoot = mkdtempSync(join(tmpdir(), 'okringest-'));
  const inbox = join(bundleRoot, 'innboks');
  mkdirSync(inbox, { recursive: true });
  try {
    await fn({ bundleRoot, inbox });
  } finally {
    rmSync(bundleRoot, { recursive: true, force: true });
  }
}

function sha256(buf) {
  return createHash('sha256').update(buf).digest('hex');
}

// Rekursivt sha256-manifest av bundlen: relativ sti -> hash (deterministisk sortert).
function manifest(root) {
  const out = {};
  const walk = (dir) => {
    for (const e of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
      const p = join(dir, e.name);
      if (e.isDirectory()) walk(p);
      else out[relative(root, p)] = sha256(readFileSync(p));
    }
  };
  walk(root);
  return out;
}

// Alle .md-filer under root (rekursivt), relativ sti.
function mdFiles(root) {
  const out = [];
  const walk = (dir) => {
    for (const e of readdirSync(dir, { withFileTypes: true })) {
      const p = join(dir, e.name);
      if (e.isDirectory()) walk(p);
      else if (e.name.endsWith('.md')) out.push(relative(root, p));
    }
  };
  walk(root);
  return out;
}

function runCheck(root, ...extra) {
  try {
    const stdout = execFileSync('node', [CHECK, root, ...extra], { encoding: 'utf8' });
    return { status: 0, stdout };
  } catch (e) {
    return { status: e.status ?? 1, stdout: `${e.stdout || ''}${e.stderr || ''}` };
  }
}

function dropTxtFixtures(inbox) {
  copyFileSync(join(SAMPLE, 'dok-a.txt'), join(inbox, 'dok-a.txt'));
  copyFileSync(join(SAMPLE, 'dok-b.txt'), join(inbox, 'dok-b.txt'));
}

test('ingest: .txt-only full pipeline zero-dep + konformitet + relasjon + original-bevaring (1,2,4,5,6)', async () => {
  await withBundle(async ({ bundleRoot, inbox }) => {
    dropTxtFixtures(inbox);
    const originalsBefore = {
      a: sha256(readFileSync(join(inbox, 'dok-a.txt'))),
      b: sha256(readFileSync(join(inbox, 'dok-b.txt'))),
    };

    const run1 = await ingestInbox(inbox, bundleRoot);
    assert.equal(run1.failed.length, 0, `ingen feilede dokumenter: ${JSON.stringify(run1.failed)}`);
    assert.equal(run1.ingested.length, 2, 'begge .txt-dokumenter ingested');
    const m1 = manifest(bundleRoot);

    // (1) Idempotens: run2 paa ULIK veggklokke -> byte-identisk bundle.
    const run2 = await ingestInbox(inbox, bundleRoot);
    assert.equal(run2.failed.length, 0, 'run2 feiler ikke');
    assert.deepEqual(manifest(bundleRoot), m1, 'sha256-manifest run1 vs run2 identisk (idempotens by construction)');

    // (2) Konformitet: default OG strict-ingest exit 0 (subprosess, kontrakt).
    assert.equal(runCheck(bundleRoot).status, 0, 'okf-check default exit 0');
    assert.equal(runCheck(bundleRoot, '--strict-ingest').status, 0, 'okf-check --strict-ingest exit 0');

    // (4) Original-bevaring: drop-zonen er byte-uendret.
    assert.equal(sha256(readFileSync(join(inbox, 'dok-a.txt'))), originalsBefore.a, 'dok-a.txt uendret');
    assert.equal(sha256(readFileSync(join(inbox, 'dok-b.txt'))), originalsBefore.b, 'dok-b.txt uendret');

    // (5) >= 1 relasjon: kryss-dokument tittel-omtale emitteres som trygg lenke.
    const all = mdFiles(bundleRoot)
      .map((f) => readFileSync(join(bundleRoot, f), 'utf8'))
      .join('\n');
    assert.match(all, /\]\(\/[^)]*\.md\)/, 'minst en bundle-root-relativ .md-relasjon emittert');
    assert.match(all, /## Relaterte dokumenter/, 'relasjons-seksjon skrevet');
  });
});

test('ingest: index-integritet -- regen byte-identisk + rot-index lister ikke innboks/ (3)', async () => {
  await withBundle(async ({ bundleRoot, inbox }) => {
    dropTxtFixtures(inbox);
    await ingestInbox(inbox, bundleRoot);

    const rootIndex = readFileSync(join(bundleRoot, 'index.md'), 'utf8');
    assert.doesNotMatch(rootIndex, /innboks/, 'rot-index lister ikke drop-zonen');

    const { generateIndexes } = await import('../scripts/okf-index.mjs');
    const before = manifest(bundleRoot);
    generateIndexes(bundleRoot);
    assert.deepEqual(manifest(bundleRoot), before, 'generateIndexes re-kjoert er byte-identisk');
  });
});

test('ingest: strict-gaten scopes til kjoeringens filer -- kuratert innhold feller ikke (7)', async () => {
  await withBundle(async ({ bundleRoot, inbox }) => {
    // Pre-eksisterende kuratert fil: out-of-vocab-type + ekstern lenke er LOVLIG
    // paa lese-siden -- den skal aldri felle ingestion (B2).
    const curatedPath = join(bundleRoot, 'syklus', 'kuratert.md');
    mkdirSync(dirname(curatedPath), { recursive: true });
    const curated = '---\ntype: Egendefinert\ntitle: Kuratert notat\n---\n'
      + 'Se [veilederen](https://www.regjeringen.no/veileder) for detaljer.\n';
    writeFileSync(curatedPath, curated);

    dropTxtFixtures(inbox);
    const run = await ingestInbox(inbox, bundleRoot);
    assert.equal(run.failed.length, 0, 'kuratert fil feller ikke kjoeringen (scoped strict)');
    assert.equal(run.ingested.length, 2, 'begge dokumenter ingested');
    assert.equal(readFileSync(curatedPath, 'utf8'), curated, 'kuratert fil byte-uroert');
  });
});

test('ingest: per-dokument-staging/gate -- fiendtlig dok discardes alene, uten spor (8)', async () => {
  await withBundle(async ({ bundleRoot, inbox }) => {
    dropTxtFixtures(inbox);
    // Dangling bundle-lenke overlever noeytralisering (trygg FORM, mangler paa
    // disk) -> strict-gaten feller dokumentet deterministisk.
    writeFileSync(
      join(inbox, 'fiendtlig.txt'),
      '# Fiendtlig notat\n\nSe [detaljer](/dokumenter/finnes-ikke.md) for mer.\n',
    );

    const run = await ingestInbox(inbox, bundleRoot);
    assert.equal(run.failed.length, 1, 'noeyaktig ett dokument discardet');
    assert.match(run.failed[0].source, /fiendtlig/, 'det fiendtlige dokumentet');
    assert.equal(run.ingested.length, 2, 'de to legitime dokumentene bestaar (Map-Reduce-isolasjon)');

    // Ingen spor: verken konsept-fil, peker-fil eller index-oppfoering.
    const rest = mdFiles(bundleRoot).map((f) => `${f}\n${readFileSync(join(bundleRoot, f), 'utf8')}`).join('\n');
    assert.doesNotMatch(rest, /fiendtlig/i, 'ingen spor av discardet dokument i tre/index');

    // Treet som bestaar er fortsatt konformt (default + strict).
    assert.equal(runCheck(bundleRoot).status, 0, 'okf-check default exit 0 etter discard');
    assert.equal(runCheck(bundleRoot, '--strict-ingest').status, 0, 'strict exit 0 etter discard');
  });
});

test('ingest: ukjent extension skippes med notice, feller ikke kjoeringen', async () => {
  await withBundle(async ({ bundleRoot, inbox }) => {
    dropTxtFixtures(inbox);
    writeFileSync(join(inbox, 'regneark.xlsx'), 'ikke-stoettet');

    const notices = [];
    const run = await ingestInbox(inbox, bundleRoot, { onNotice: (m) => notices.push(m) });
    assert.equal(run.skipped.length, 1, 'xlsx skippet');
    assert.equal(run.failed.length, 0, 'skip er ikke feil');
    assert.equal(run.ingested.length, 2, 'txt-dokumentene ingested');
    assert.ok(notices.some((n) => n.includes('xlsx')), 'norsk notice om ukjent filtype');
  });
});

test('ingest: binaer-original (.docx/.pdf) passerer strict -- pekerfil uten lenkeform (B1-aksept)', { skip: !engines }, async () => {
  await withBundle(async ({ bundleRoot, inbox }) => {
    copyFileSync(join(SAMPLE, 'dok.docx'), join(inbox, 'dok.docx'));
    copyFileSync(join(SAMPLE, 'dok.pdf'), join(inbox, 'dok.pdf'));

    const run = await ingestInbox(inbox, bundleRoot);
    assert.equal(run.failed.length, 0, `binaer-ingest feiler ikke: ${JSON.stringify(run.failed)}`);
    assert.equal(run.ingested.length, 2, 'begge binaer-dokumenter ingested');

    // B1-aksept: hele bundlen passerer strict med ikke-.md-originaler.
    assert.equal(runCheck(bundleRoot, '--strict-ingest').status, 0, 'strict exit 0 med .docx/.pdf-originaler');

    // Pekerfiler finnes og baerer original-stien i resource (uten lenkeform).
    const pointers = mdFiles(bundleRoot).filter((f) => f.endsWith('.kilde.md'));
    assert.equal(pointers.length, 2, 'en pekerfil per original');
    for (const p of pointers) {
      const content = readFileSync(join(bundleRoot, p), 'utf8');
      assert.match(content, /^resource: innboks\/dok[^\n]*$/m, 'resource baerer original-stien');
      assert.doesNotMatch(content, /\]\(/, 'ingen lenkeform i pekerfil (B1)');
    }

    // Idempotens gjelder ogsaa binaer-stien (convert-twice-identical + mtime).
    const m1 = manifest(bundleRoot);
    await ingestInbox(inbox, bundleRoot);
    assert.deepEqual(manifest(bundleRoot), m1, 'binaer-ingest idempotent');
  });
});