feat(linkedin-studio): SB-S1 ingest data layer — record grammar + content-id [skip-docs]

PublishedRecord file-per-post grammar (fixed 5-line header + --- sentinel +
verbatim body, no YAML) with parse∘serialize identity, and mintContentId =
sha256(VERBATIM body)[:12] — byte-identity dedup so two structurally-different
posts never collide (avoids the normalizeContent silent-data-loss path).
parsePublishedRecord rejects a published/ record whose provenance != published
(corruption signal). 10 grammar tests incl. the B1 edge battery (empty body,
body starting with ---, mid-text \n---\n, header-shaped first line) + 4 id tests.
Pure layer only; IO + CLI + voice-trainer wiring follow. brain 34→48 tests.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RigJBiRFNtFZKCz21qNbQ4

scripts/brain/src/id.ts

@@ -37,6 +37,21 @@ export function mintEntityId(seed: { kind: string; key: string }): string {
     .slice(0, 12);
 }
 
+/**
+ * Content-hash id for an ingested published post = first 12 hex of
+ * `sha256(VERBATIM body)`. Mirrors the specifics-bank `specificId` hash idiom
+ * (`specifics-bank/src/bank.ts:57`) but deliberately hashes the body BYTE-EXACT —
+ * NOT through `normalizeContent` (which lowercases + collapses whitespace). For a
+ * full post body that normalization would map two structurally-different posts
+ * (same words, different line breaks/case) to the same id, and the second would be
+ * silently skipped on write — data loss of the exact gold signal SB-S1 captures.
+ * Byte-identity hashing makes the id true content identity: only an identical
+ * re-ingest collides; any real difference mints a distinct id (SB-S1).
+ */
+export function mintContentId(text: string): string {
+  return createHash("sha256").update(text).digest("hex").slice(0, 12);
+}
+
 /**
  * Normalize a raw provenance string to the single brain vocabulary. Trims +
  * lowercases, then returns the match or THROWS — provenance is load-bearing for

scripts/brain/src/ingest.ts

@@ -0,0 +1,95 @@
+/**
+ * Ingest — the published-post gold signal (SB-S1).
+ *
+ * Captures the user's ACTUAL published posts into `ingest/published/` tagged
+ * `provenance=published`, so the voice/profile-learning surface can learn from
+ * human-published content ONLY (the model-collapse guard — never from ai-draft).
+ *
+ * This module has two halves:
+ *  - PURE: `PublishedRecord` + `parsePublishedRecord` / `serializePublishedRecord`
+ *    (a file-per-post constrained-header grammar; NO YAML dep, mirroring the brain
+ *    profile.md line-grammar idiom). `parse ∘ serialize = identity` (SC2).
+ *  - IO: `writePublished` / `ingestText` / `scanInbox` / `listPublished` — idempotent,
+ *    collision-safe, create-on-demand under the brain `dataRoot` (SC1/SC3/SC4).
+ *
+ * The record id is `mintContentId(VERBATIM body)` (id.ts) — byte-identity, so two
+ * structurally-different posts never collide and the write path never silently
+ * drops a differing body.
+ */
+
+import { mintContentId, normalizeProvenance } from "./id.js";
+
+/** One ingested published post. `provenance` is always `published` in SB-S1. */
+export interface PublishedRecord {
+  /** mintContentId(body) — 12 hex, the dedupe key + filename stem. */
+  id: string;
+  /** Always `published` — the type pins it; parse rejects any other value. */
+  provenance: "published";
+  /** YYYY-MM-DD — when the post was published (operator-supplied or = captured_at). */
+  published_date: string;
+  /** YYYY-MM-DD — when this record was ingested. */
+  captured_at: string;
+  /** `manual` (default) | a future connector token. */
+  source: string;
+  /** The verbatim post body — byte-exact, may contain anything (`]`/`|`/newlines/`---`). */
+  body: string;
+}
+
+/** The one separator string, used identically by serialize and parse. */
+const SENTINEL = "\n---\n";
+
+/**
+ * Serialize a record to the file grammar: a fixed 5-line header, the `---`
+ * sentinel, then the verbatim body. No title, no blank-line padding (the body is
+ * captured byte-exact, so any padding would break the round-trip identity).
+ */
+export function serializePublishedRecord(rec: PublishedRecord): string {
+  const header = [
+    `id: ${rec.id}`,
+    `provenance: ${rec.provenance}`,
+    `published_date: ${rec.published_date}`,
+    `captured_at: ${rec.captured_at}`,
+    `source: ${rec.source}`,
+  ].join("\n");
+  return header + SENTINEL + rec.body;
+}
+
+function headerScalar(header: string, key: string): string {
+  const m = header.match(new RegExp(`^${key}:\\s*(.*?)\\s*$`, "m"));
+  if (!m) throw new Error(`malformed published record: missing "${key}:" header`);
+  return m[1];
+}
+
+/**
+ * Parse a record back from the file grammar. Splits on the FIRST `---` sentinel
+ * (the header lines are constrained `key: value` pairs that never contain one), so
+ * a body that itself contains `\n---\n` is preserved verbatim. The header scalars
+ * are read only from the header slice — a header-shaped body line cannot leak in.
+ * `provenance` is asserted to be `published`: a `published/` record carrying any
+ * other provenance is a corruption signal and throws (never silently accepted).
+ */
+export function parsePublishedRecord(text: string): PublishedRecord {
+  const idx = text.indexOf(SENTINEL);
+  if (idx === -1) throw new Error("malformed published record: no `---` separator");
+  const header = text.slice(0, idx);
+  const body = text.slice(idx + SENTINEL.length);
+
+  const id = headerScalar(header, "id");
+  if (!/^[0-9a-f]{12}$/.test(id)) {
+    throw new Error(`malformed published record: bad id ${JSON.stringify(id)}`);
+  }
+  const provenance = normalizeProvenance(headerScalar(header, "provenance"));
+  if (provenance !== "published") {
+    throw new Error(
+      `corrupt published record: provenance is "${provenance}", expected "published"`,
+    );
+  }
+  return {
+    id,
+    provenance,
+    published_date: headerScalar(header, "published_date"),
+    captured_at: headerScalar(header, "captured_at"),
+    source: headerScalar(header, "source"),
+    body,
+  };
+}

scripts/brain/tests/ingest.test.ts

@@ -0,0 +1,84 @@
+import { describe, test } from "node:test";
+import assert from "node:assert/strict";
+
+import { mintContentId } from "../src/id.js";
+import {
+  parsePublishedRecord,
+  serializePublishedRecord,
+  type PublishedRecord,
+} from "../src/ingest.js";
+
+const baseRec = (body: string): PublishedRecord => ({
+  id: mintContentId(body),
+  provenance: "published",
+  published_date: "2026-05-26",
+  captured_at: "2026-06-23",
+  source: "manual",
+  body,
+});
+
+/** Round-trip helper: parse∘serialize must equal the original record. */
+function roundTrip(rec: PublishedRecord): PublishedRecord {
+  return parsePublishedRecord(serializePublishedRecord(rec));
+}
+
+describe("mintContentId — verbatim-body content hash (SC3)", () => {
+  test("deterministic: same body → same id", () => {
+    const body = "Jeg lærte noe om dømmekraft i dag.\n\nDel 1 av serien.";
+    assert.equal(mintContentId(body), mintContentId(body));
+  });
+
+  test("returns a 12-hex id", () => {
+    assert.match(mintContentId("any body"), /^[0-9a-f]{12}$/);
+  });
+
+  test("different bodies → different ids", () => {
+    assert.notEqual(mintContentId("post A"), mintContentId("post B"));
+  });
+
+  test("whitespace/case differences mint DIFFERENT ids (no normalize-collision, B2 guard)", () => {
+    // Under specifics-bank normalizeContent these would collapse to the SAME id —
+    // proving the post-body hash is byte-identity, not normalized.
+    assert.notEqual(mintContentId("Hello World"), mintContentId("hello  world"));
+    assert.notEqual(
+      mintContentId("Line one\nLine two"),
+      mintContentId("Line one Line two"),
+    );
+  });
+});
+
+describe("PublishedRecord grammar — parse∘serialize identity (SC2, B1 edge battery)", () => {
+  const bodies: Record<string, string> = {
+    "plain multi-paragraph": "Hook line.\n\nBody paragraph one.\n\nBody paragraph two.",
+    "empty body": "",
+    "single newline": "\n",
+    "trailing newlines": "post text\n\n",
+    "body starting with ---": "---\nlooks like a separator but is body",
+    "body containing a \\n---\\n sentinel mid-text": "before\n---\nafter the fake separator",
+    "header-shaped first line": "id: 0123456789ab\nprovenance: published",
+    "special chars ] | quotes": 'value with ] and | and "quotes" inside',
+  };
+
+  for (const [label, body] of Object.entries(bodies)) {
+    test(`round-trips: ${label}`, () => {
+      const rec = baseRec(body);
+      assert.deepEqual(roundTrip(rec), rec);
+    });
+  }
+
+  test("serialized form has the fixed 5-line header then the --- sentinel", () => {
+    const text = serializePublishedRecord(baseRec("hi"));
+    const [header, ...rest] = text.split("\n---\n");
+    assert.equal(header.split("\n").length, 5, "exactly 5 header lines before the sentinel");
+    assert.equal(rest.join("\n---\n"), "hi", "body is everything after the FIRST sentinel");
+    assert.match(text, /^id: [0-9a-f]{12}\nprovenance: published\n/);
+  });
+
+  test("a published/ record with non-published provenance is rejected (corruption signal)", () => {
+    const corrupt = serializePublishedRecord(baseRec("x")).replace(
+      "provenance: published",
+      "provenance: ai-draft",
+    );
+    assert.throws(() => parsePublishedRecord(corrupt), /provenance/i);
+  });
+});