feat(linkedin-studio): SB-S1 ingest IO — writePublished/ingestText/scanInbox/listPublished [skip-docs]
Idempotent, collision-safe, create-on-demand IO under the brain dataRoot: - writePublished: identical body → no-op; differing body at same id → disambiguated <id>-N.md (never clobber, never silently drop a gold record, B2 defence-in-depth). - ingestText: provenance always published; published_date defaults to captured_at. - scanInbox: top-level *.md only (skips .DS_Store/dotfiles/non-md), non-destructive, empty/absent inbox = clean no-op, re-scan re-skips via dedup. - listPublished: parses each file in try/catch (malformed → skipped count, never crashes), surfaces provenance so ai-draft leakage is eyeball-visible. 9 temp-dir IO tests. brain 48→57 tests, tsc clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RigJBiRFNtFZKCz21qNbQ4
This commit is contained in:
parent
3e3990f36e
commit
0d3e4911d7
2 changed files with 277 additions and 0 deletions
|
|
@ -17,6 +17,10 @@
|
|||
* drops a differing body.
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { dataRoot } from "./dataRoot.js";
|
||||
import { mintContentId, normalizeProvenance } from "./id.js";
|
||||
|
||||
/** One ingested published post. `provenance` is always `published` in SB-S1. */
|
||||
|
|
@ -93,3 +97,148 @@ export function parsePublishedRecord(text: string): PublishedRecord {
|
|||
body,
|
||||
};
|
||||
}
|
||||
|
||||
// ── IO ───────────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// All IO resolves through the brain package's own `dataRoot` (LINKEDIN_STUDIO_DATA
|
||||
// overrides the root for tests + installs). Create-on-demand: `ingest/published/`
|
||||
// is mkdir'd as needed, so ingest does NOT depend on a prior `brain init`.
|
||||
|
||||
const PUBLISHED_SUB = join("ingest", "published");
|
||||
const INBOX_SUB = join("ingest", "inbox");
|
||||
|
||||
/** The body stored at `path`, or `null` if it can't be read/parsed (treat as differing). */
|
||||
function storedBody(path: string): string | null {
|
||||
try {
|
||||
return parsePublishedRecord(readFileSync(path, "utf8")).body;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export interface WriteResult {
|
||||
/** true iff a file was written (false = identical record already present). */
|
||||
written: boolean;
|
||||
/** The path written or matched. */
|
||||
path: string;
|
||||
/** true iff an id-collision with a DIFFERING body forced a disambiguated name. */
|
||||
collision?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a record idempotently + collision-safely. If `<id>.md` already holds the
|
||||
* SAME body → no-op (`written:false`). If it holds a DIFFERENT body (a 48-bit hash
|
||||
* collision or an unreadable file) → write the next free `<id>-N.md` rather than
|
||||
* clobber or silently skip — so a differing gold record is never lost (B2).
|
||||
*/
|
||||
export function writePublished(rec: PublishedRecord): WriteResult {
|
||||
const dir = dataRoot(PUBLISHED_SUB);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const text = serializePublishedRecord(rec);
|
||||
|
||||
const primary = join(dir, `${rec.id}.md`);
|
||||
if (!existsSync(primary)) {
|
||||
writeFileSync(primary, text, "utf8");
|
||||
return { written: true, path: primary };
|
||||
}
|
||||
if (storedBody(primary) === rec.body) return { written: false, path: primary };
|
||||
|
||||
for (let n = 2; ; n++) {
|
||||
const cand = join(dir, `${rec.id}-${n}.md`);
|
||||
if (!existsSync(cand)) {
|
||||
writeFileSync(cand, text, "utf8");
|
||||
return { written: true, path: cand, collision: true };
|
||||
}
|
||||
if (storedBody(cand) === rec.body) return { written: false, path: cand, collision: true };
|
||||
}
|
||||
}
|
||||
|
||||
export interface IngestResult extends WriteResult {
|
||||
record: PublishedRecord;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mint a record from a verbatim post body and write it. `provenance` is always
|
||||
* `published`; `published_date` defaults to `captured_at` when not supplied. Pure
|
||||
* inputs (the caller supplies `captured_at`), idempotent + collision-safe write.
|
||||
*/
|
||||
export function ingestText(opts: {
|
||||
body: string;
|
||||
captured_at: string;
|
||||
source?: string;
|
||||
published_date?: string;
|
||||
}): IngestResult {
|
||||
const record: PublishedRecord = {
|
||||
id: mintContentId(opts.body),
|
||||
provenance: "published",
|
||||
published_date: opts.published_date ?? opts.captured_at,
|
||||
captured_at: opts.captured_at,
|
||||
source: opts.source ?? "manual",
|
||||
body: opts.body,
|
||||
};
|
||||
return { record, ...writePublished(record) };
|
||||
}
|
||||
|
||||
export interface ScanResult {
|
||||
/** ids of records written this scan. */
|
||||
processed: string[];
|
||||
/** ids of records already present (skipped as duplicates). */
|
||||
skipped: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the inbox drop-zone into `published/`. Reads TOP-LEVEL `*.md` files only,
|
||||
* skipping dotfiles (`.DS_Store` etc.) and non-markdown — no recursion. An empty or
|
||||
* absent inbox is a clean no-op. Non-destructive in SB-S1: processed inbox files are
|
||||
* left in place (a re-scan re-skips via the published dedup).
|
||||
*/
|
||||
export function scanInbox(opts: { captured_at: string; source?: string }): ScanResult {
|
||||
const dir = dataRoot(INBOX_SUB);
|
||||
const processed: string[] = [];
|
||||
const skipped: string[] = [];
|
||||
if (!existsSync(dir)) return { processed, skipped };
|
||||
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
if (!entry.isFile() || entry.name.startsWith(".") || !entry.name.endsWith(".md")) continue;
|
||||
const body = readFileSync(join(dir, entry.name), "utf8");
|
||||
const res = ingestText({ body, captured_at: opts.captured_at, source: opts.source });
|
||||
(res.written ? processed : skipped).push(res.record.id);
|
||||
}
|
||||
return { processed, skipped };
|
||||
}
|
||||
|
||||
export interface PublishedSummary {
|
||||
id: string;
|
||||
provenance: string;
|
||||
published_date: string;
|
||||
firstLine: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* List the published gold corpus. Each file is parsed inside a try/catch — a
|
||||
* malformed/hand-dropped file is counted as `skipped`, never crashing the command.
|
||||
* Surfaces `provenance` so the operator can eyeball that nothing `ai-draft` leaked.
|
||||
*/
|
||||
export function listPublished(): { records: PublishedSummary[]; skipped: number } {
|
||||
const dir = dataRoot(PUBLISHED_SUB);
|
||||
const records: PublishedSummary[] = [];
|
||||
let skipped = 0;
|
||||
if (!existsSync(dir)) return { records, skipped };
|
||||
|
||||
for (const name of readdirSync(dir)) {
|
||||
if (name.startsWith(".") || !name.endsWith(".md")) continue;
|
||||
try {
|
||||
const rec = parsePublishedRecord(readFileSync(join(dir, name), "utf8"));
|
||||
records.push({
|
||||
id: rec.id,
|
||||
provenance: rec.provenance,
|
||||
published_date: rec.published_date,
|
||||
firstLine: rec.body.split("\n", 1)[0],
|
||||
});
|
||||
} catch {
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
records.sort((a, b) => a.published_date.localeCompare(b.published_date));
|
||||
return { records, skipped };
|
||||
}
|
||||
|
|
|
|||
128
scripts/brain/tests/publish.test.ts
Normal file
128
scripts/brain/tests/publish.test.ts
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
import { describe, test, beforeEach, afterEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync, mkdirSync, readdirSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
import {
|
||||
writePublished,
|
||||
ingestText,
|
||||
scanInbox,
|
||||
listPublished,
|
||||
serializePublishedRecord,
|
||||
parsePublishedRecord,
|
||||
type PublishedRecord,
|
||||
} from "../src/ingest.js";
|
||||
import { mintContentId } from "../src/id.js";
|
||||
|
||||
const CAPTURED = "2026-06-23";
|
||||
|
||||
function publishedDir(root: string): string {
|
||||
return join(root, "ingest", "published");
|
||||
}
|
||||
function inboxDir(root: string): string {
|
||||
return join(root, "ingest", "inbox");
|
||||
}
|
||||
function mdFiles(dir: string): string[] {
|
||||
return existsSync(dir) ? readdirSync(dir).filter((f) => f.endsWith(".md")) : [];
|
||||
}
|
||||
|
||||
describe("ingest IO — writePublished / ingestText / scanInbox / listPublished (SC1/SC3/SC4)", () => {
|
||||
let root: string;
|
||||
const prevEnv = process.env.LINKEDIN_STUDIO_DATA;
|
||||
|
||||
beforeEach(() => {
|
||||
root = mkdtempSync(join(tmpdir(), "brain-ingest-"));
|
||||
process.env.LINKEDIN_STUDIO_DATA = root;
|
||||
});
|
||||
afterEach(() => {
|
||||
if (prevEnv === undefined) delete process.env.LINKEDIN_STUDIO_DATA;
|
||||
else process.env.LINKEDIN_STUDIO_DATA = prevEnv;
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("create-on-demand: ingestText writes ingest/published/<id>.md with no prior init (SC1)", () => {
|
||||
const body = "Min første publiserte post.\n\nMed avsnitt.";
|
||||
const res = ingestText({ body, captured_at: CAPTURED });
|
||||
assert.equal(res.written, true);
|
||||
const path = join(publishedDir(root), `${mintContentId(body)}.md`);
|
||||
assert.ok(existsSync(path), "record file created at runtime data-path");
|
||||
const rec = parsePublishedRecord(readFileSync(path, "utf8"));
|
||||
assert.equal(rec.provenance, "published");
|
||||
assert.equal(rec.body, body, "verbatim body preserved");
|
||||
});
|
||||
|
||||
test("published_date defaults to captured_at when not supplied", () => {
|
||||
const res = ingestText({ body: "no date given", captured_at: CAPTURED });
|
||||
assert.equal(res.record.published_date, CAPTURED);
|
||||
});
|
||||
|
||||
test("published_date is kept when supplied", () => {
|
||||
const res = ingestText({ body: "dated", captured_at: CAPTURED, published_date: "2026-05-26" });
|
||||
assert.equal(res.record.published_date, "2026-05-26");
|
||||
});
|
||||
|
||||
test("re-ingesting an identical body is a no-op (SC3 dedup): {written:false}, no duplicate file", () => {
|
||||
const body = "same exact text";
|
||||
ingestText({ body, captured_at: CAPTURED });
|
||||
const res2 = ingestText({ body, captured_at: "2026-07-01" }); // different day, same body
|
||||
assert.equal(res2.written, false);
|
||||
assert.equal(mdFiles(publishedDir(root)).length, 1, "still exactly one record");
|
||||
});
|
||||
|
||||
test("id-collision with a DIFFERING body disambiguates to <id>-2.md — never silent data loss (B2)", () => {
|
||||
const body = "the real body";
|
||||
const id = mintContentId(body);
|
||||
// Pre-plant a file at <id>.md whose body is DIFFERENT (simulates a hash collision).
|
||||
mkdirSync(publishedDir(root), { recursive: true });
|
||||
const squatter: PublishedRecord = {
|
||||
id, provenance: "published", published_date: CAPTURED, captured_at: CAPTURED,
|
||||
source: "manual", body: "a DIFFERENT body that happens to share the id",
|
||||
};
|
||||
writeFileSync(join(publishedDir(root), `${id}.md`), serializePublishedRecord(squatter), "utf8");
|
||||
|
||||
const res = writePublished({ ...squatter, body });
|
||||
assert.equal(res.written, true);
|
||||
assert.equal(res.collision, true);
|
||||
assert.match(res.path, /-2\.md$/);
|
||||
assert.equal(mdFiles(publishedDir(root)).length, 2, "both records kept — no data loss");
|
||||
});
|
||||
|
||||
test("scanInbox processes top-level *.md only, skipping dotfiles like .DS_Store (SC4)", () => {
|
||||
mkdirSync(inboxDir(root), { recursive: true });
|
||||
writeFileSync(join(inboxDir(root), "post-a.md"), "Post A body", "utf8");
|
||||
writeFileSync(join(inboxDir(root), "post-b.md"), "Post B body", "utf8");
|
||||
writeFileSync(join(inboxDir(root), ".DS_Store"), "junk", "utf8");
|
||||
writeFileSync(join(inboxDir(root), "notes.txt"), "not markdown", "utf8");
|
||||
|
||||
const res = scanInbox({ captured_at: CAPTURED });
|
||||
assert.equal(res.processed.length, 2, "only the two .md files processed");
|
||||
assert.equal(mdFiles(publishedDir(root)).length, 2);
|
||||
});
|
||||
|
||||
test("scanInbox on an empty/absent inbox is a clean no-op (SC4)", () => {
|
||||
const res = scanInbox({ captured_at: CAPTURED });
|
||||
assert.deepEqual(res, { processed: [], skipped: [] });
|
||||
});
|
||||
|
||||
test("re-running scanInbox re-skips already-published records (idempotent)", () => {
|
||||
mkdirSync(inboxDir(root), { recursive: true });
|
||||
writeFileSync(join(inboxDir(root), "post.md"), "once", "utf8");
|
||||
scanInbox({ captured_at: CAPTURED });
|
||||
const res2 = scanInbox({ captured_at: CAPTURED });
|
||||
assert.equal(res2.processed.length, 0);
|
||||
assert.equal(res2.skipped.length, 1);
|
||||
assert.equal(mdFiles(publishedDir(root)).length, 1, "no duplicate");
|
||||
});
|
||||
|
||||
test("listPublished surfaces provenance and skips a malformed file without throwing", () => {
|
||||
ingestText({ body: "good record", captured_at: CAPTURED });
|
||||
mkdirSync(publishedDir(root), { recursive: true });
|
||||
writeFileSync(join(publishedDir(root), "broken.md"), "not a valid record at all", "utf8");
|
||||
|
||||
const res = listPublished();
|
||||
assert.equal(res.records.length, 1, "the one valid record listed");
|
||||
assert.equal(res.records[0].provenance, "published");
|
||||
assert.equal(res.skipped, 1, "the malformed file counted as skipped, not a crash");
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue