From 0d3e4911d7389291039182afced0bbac42efa9b0 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 14:54:31 +0200 Subject: [PATCH] =?UTF-8?q?feat(linkedin-studio):=20SB-S1=20ingest=20IO=20?= =?UTF-8?q?=E2=80=94=20writePublished/ingestText/scanInbox/listPublished?= =?UTF-8?q?=20[skip-docs]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Idempotent, collision-safe, create-on-demand IO under the brain dataRoot: - writePublished: identical body → no-op; differing body at same id → disambiguated -N.md (never clobber, never silently drop a gold record, B2 defence-in-depth). - ingestText: provenance always published; published_date defaults to captured_at. - scanInbox: top-level *.md only (skips .DS_Store/dotfiles/non-md), non-destructive, empty/absent inbox = clean no-op, re-scan re-skips via dedup. - listPublished: parses each file in try/catch (malformed → skipped count, never crashes), surfaces provenance so ai-draft leakage is eyeball-visible. 9 temp-dir IO tests. brain 48→57 tests, tsc clean. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01RigJBiRFNtFZKCz21qNbQ4 --- scripts/brain/src/ingest.ts | 149 ++++++++++++++++++++++++++++ scripts/brain/tests/publish.test.ts | 128 ++++++++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 scripts/brain/tests/publish.test.ts diff --git a/scripts/brain/src/ingest.ts b/scripts/brain/src/ingest.ts index 5307e42..a59fcce 100644 --- a/scripts/brain/src/ingest.ts +++ b/scripts/brain/src/ingest.ts @@ -17,6 +17,10 @@ * drops a differing body. */ +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +import { dataRoot } from "./dataRoot.js"; import { mintContentId, normalizeProvenance } from "./id.js"; /** One ingested published post. `provenance` is always `published` in SB-S1. */ @@ -93,3 +97,148 @@ export function parsePublishedRecord(text: string): PublishedRecord { body, }; } + +// ── IO ─────────────────────────────────────────────────────────────────────── +// +// All IO resolves through the brain package's own `dataRoot` (LINKEDIN_STUDIO_DATA +// overrides the root for tests + installs). Create-on-demand: `ingest/published/` +// is mkdir'd as needed, so ingest does NOT depend on a prior `brain init`. + +const PUBLISHED_SUB = join("ingest", "published"); +const INBOX_SUB = join("ingest", "inbox"); + +/** The body stored at `path`, or `null` if it can't be read/parsed (treat as differing). */ +function storedBody(path: string): string | null { + try { + return parsePublishedRecord(readFileSync(path, "utf8")).body; + } catch { + return null; + } +} + +export interface WriteResult { + /** true iff a file was written (false = identical record already present). */ + written: boolean; + /** The path written or matched. */ + path: string; + /** true iff an id-collision with a DIFFERING body forced a disambiguated name. */ + collision?: boolean; +} + +/** + * Write a record idempotently + collision-safely. If `.md` already holds the + * SAME body → no-op (`written:false`). If it holds a DIFFERENT body (a 48-bit hash + * collision or an unreadable file) → write the next free `-N.md` rather than + * clobber or silently skip — so a differing gold record is never lost (B2). + */ +export function writePublished(rec: PublishedRecord): WriteResult { + const dir = dataRoot(PUBLISHED_SUB); + mkdirSync(dir, { recursive: true }); + const text = serializePublishedRecord(rec); + + const primary = join(dir, `${rec.id}.md`); + if (!existsSync(primary)) { + writeFileSync(primary, text, "utf8"); + return { written: true, path: primary }; + } + if (storedBody(primary) === rec.body) return { written: false, path: primary }; + + for (let n = 2; ; n++) { + const cand = join(dir, `${rec.id}-${n}.md`); + if (!existsSync(cand)) { + writeFileSync(cand, text, "utf8"); + return { written: true, path: cand, collision: true }; + } + if (storedBody(cand) === rec.body) return { written: false, path: cand, collision: true }; + } +} + +export interface IngestResult extends WriteResult { + record: PublishedRecord; +} + +/** + * Mint a record from a verbatim post body and write it. `provenance` is always + * `published`; `published_date` defaults to `captured_at` when not supplied. Pure + * inputs (the caller supplies `captured_at`), idempotent + collision-safe write. + */ +export function ingestText(opts: { + body: string; + captured_at: string; + source?: string; + published_date?: string; +}): IngestResult { + const record: PublishedRecord = { + id: mintContentId(opts.body), + provenance: "published", + published_date: opts.published_date ?? opts.captured_at, + captured_at: opts.captured_at, + source: opts.source ?? "manual", + body: opts.body, + }; + return { record, ...writePublished(record) }; +} + +export interface ScanResult { + /** ids of records written this scan. */ + processed: string[]; + /** ids of records already present (skipped as duplicates). */ + skipped: string[]; +} + +/** + * Process the inbox drop-zone into `published/`. Reads TOP-LEVEL `*.md` files only, + * skipping dotfiles (`.DS_Store` etc.) and non-markdown — no recursion. An empty or + * absent inbox is a clean no-op. Non-destructive in SB-S1: processed inbox files are + * left in place (a re-scan re-skips via the published dedup). + */ +export function scanInbox(opts: { captured_at: string; source?: string }): ScanResult { + const dir = dataRoot(INBOX_SUB); + const processed: string[] = []; + const skipped: string[] = []; + if (!existsSync(dir)) return { processed, skipped }; + + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (!entry.isFile() || entry.name.startsWith(".") || !entry.name.endsWith(".md")) continue; + const body = readFileSync(join(dir, entry.name), "utf8"); + const res = ingestText({ body, captured_at: opts.captured_at, source: opts.source }); + (res.written ? processed : skipped).push(res.record.id); + } + return { processed, skipped }; +} + +export interface PublishedSummary { + id: string; + provenance: string; + published_date: string; + firstLine: string; +} + +/** + * List the published gold corpus. Each file is parsed inside a try/catch — a + * malformed/hand-dropped file is counted as `skipped`, never crashing the command. + * Surfaces `provenance` so the operator can eyeball that nothing `ai-draft` leaked. + */ +export function listPublished(): { records: PublishedSummary[]; skipped: number } { + const dir = dataRoot(PUBLISHED_SUB); + const records: PublishedSummary[] = []; + let skipped = 0; + if (!existsSync(dir)) return { records, skipped }; + + for (const name of readdirSync(dir)) { + if (name.startsWith(".") || !name.endsWith(".md")) continue; + try { + const rec = parsePublishedRecord(readFileSync(join(dir, name), "utf8")); + records.push({ + id: rec.id, + provenance: rec.provenance, + published_date: rec.published_date, + firstLine: rec.body.split("\n", 1)[0], + }); + } catch { + skipped++; + } + } + records.sort((a, b) => a.published_date.localeCompare(b.published_date)); + return { records, skipped }; +} diff --git a/scripts/brain/tests/publish.test.ts b/scripts/brain/tests/publish.test.ts new file mode 100644 index 0000000..bc77c8f --- /dev/null +++ b/scripts/brain/tests/publish.test.ts @@ -0,0 +1,128 @@ +import { describe, test, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync, mkdirSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +import { + writePublished, + ingestText, + scanInbox, + listPublished, + serializePublishedRecord, + parsePublishedRecord, + type PublishedRecord, +} from "../src/ingest.js"; +import { mintContentId } from "../src/id.js"; + +const CAPTURED = "2026-06-23"; + +function publishedDir(root: string): string { + return join(root, "ingest", "published"); +} +function inboxDir(root: string): string { + return join(root, "ingest", "inbox"); +} +function mdFiles(dir: string): string[] { + return existsSync(dir) ? readdirSync(dir).filter((f) => f.endsWith(".md")) : []; +} + +describe("ingest IO — writePublished / ingestText / scanInbox / listPublished (SC1/SC3/SC4)", () => { + let root: string; + const prevEnv = process.env.LINKEDIN_STUDIO_DATA; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "brain-ingest-")); + process.env.LINKEDIN_STUDIO_DATA = root; + }); + afterEach(() => { + if (prevEnv === undefined) delete process.env.LINKEDIN_STUDIO_DATA; + else process.env.LINKEDIN_STUDIO_DATA = prevEnv; + rmSync(root, { recursive: true, force: true }); + }); + + test("create-on-demand: ingestText writes ingest/published/.md with no prior init (SC1)", () => { + const body = "Min første publiserte post.\n\nMed avsnitt."; + const res = ingestText({ body, captured_at: CAPTURED }); + assert.equal(res.written, true); + const path = join(publishedDir(root), `${mintContentId(body)}.md`); + assert.ok(existsSync(path), "record file created at runtime data-path"); + const rec = parsePublishedRecord(readFileSync(path, "utf8")); + assert.equal(rec.provenance, "published"); + assert.equal(rec.body, body, "verbatim body preserved"); + }); + + test("published_date defaults to captured_at when not supplied", () => { + const res = ingestText({ body: "no date given", captured_at: CAPTURED }); + assert.equal(res.record.published_date, CAPTURED); + }); + + test("published_date is kept when supplied", () => { + const res = ingestText({ body: "dated", captured_at: CAPTURED, published_date: "2026-05-26" }); + assert.equal(res.record.published_date, "2026-05-26"); + }); + + test("re-ingesting an identical body is a no-op (SC3 dedup): {written:false}, no duplicate file", () => { + const body = "same exact text"; + ingestText({ body, captured_at: CAPTURED }); + const res2 = ingestText({ body, captured_at: "2026-07-01" }); // different day, same body + assert.equal(res2.written, false); + assert.equal(mdFiles(publishedDir(root)).length, 1, "still exactly one record"); + }); + + test("id-collision with a DIFFERING body disambiguates to -2.md — never silent data loss (B2)", () => { + const body = "the real body"; + const id = mintContentId(body); + // Pre-plant a file at .md whose body is DIFFERENT (simulates a hash collision). + mkdirSync(publishedDir(root), { recursive: true }); + const squatter: PublishedRecord = { + id, provenance: "published", published_date: CAPTURED, captured_at: CAPTURED, + source: "manual", body: "a DIFFERENT body that happens to share the id", + }; + writeFileSync(join(publishedDir(root), `${id}.md`), serializePublishedRecord(squatter), "utf8"); + + const res = writePublished({ ...squatter, body }); + assert.equal(res.written, true); + assert.equal(res.collision, true); + assert.match(res.path, /-2\.md$/); + assert.equal(mdFiles(publishedDir(root)).length, 2, "both records kept — no data loss"); + }); + + test("scanInbox processes top-level *.md only, skipping dotfiles like .DS_Store (SC4)", () => { + mkdirSync(inboxDir(root), { recursive: true }); + writeFileSync(join(inboxDir(root), "post-a.md"), "Post A body", "utf8"); + writeFileSync(join(inboxDir(root), "post-b.md"), "Post B body", "utf8"); + writeFileSync(join(inboxDir(root), ".DS_Store"), "junk", "utf8"); + writeFileSync(join(inboxDir(root), "notes.txt"), "not markdown", "utf8"); + + const res = scanInbox({ captured_at: CAPTURED }); + assert.equal(res.processed.length, 2, "only the two .md files processed"); + assert.equal(mdFiles(publishedDir(root)).length, 2); + }); + + test("scanInbox on an empty/absent inbox is a clean no-op (SC4)", () => { + const res = scanInbox({ captured_at: CAPTURED }); + assert.deepEqual(res, { processed: [], skipped: [] }); + }); + + test("re-running scanInbox re-skips already-published records (idempotent)", () => { + mkdirSync(inboxDir(root), { recursive: true }); + writeFileSync(join(inboxDir(root), "post.md"), "once", "utf8"); + scanInbox({ captured_at: CAPTURED }); + const res2 = scanInbox({ captured_at: CAPTURED }); + assert.equal(res2.processed.length, 0); + assert.equal(res2.skipped.length, 1); + assert.equal(mdFiles(publishedDir(root)).length, 1, "no duplicate"); + }); + + test("listPublished surfaces provenance and skips a malformed file without throwing", () => { + ingestText({ body: "good record", captured_at: CAPTURED }); + mkdirSync(publishedDir(root), { recursive: true }); + writeFileSync(join(publishedDir(root), "broken.md"), "not a valid record at all", "utf8"); + + const res = listPublished(); + assert.equal(res.records.length, 1, "the one valid record listed"); + assert.equal(res.records[0].provenance, "published"); + assert.equal(res.skipped, 1, "the malformed file counted as skipped, not a crash"); + }); +});