diff --git a/scripts/brain/src/id.ts b/scripts/brain/src/id.ts index bc0b4ab..c575cd9 100644 --- a/scripts/brain/src/id.ts +++ b/scripts/brain/src/id.ts @@ -37,6 +37,21 @@ export function mintEntityId(seed: { kind: string; key: string }): string { .slice(0, 12); } +/** + * Content-hash id for an ingested published post = first 12 hex of + * `sha256(VERBATIM body)`. Mirrors the specifics-bank `specificId` hash idiom + * (`specifics-bank/src/bank.ts:57`) but deliberately hashes the body BYTE-EXACT — + * NOT through `normalizeContent` (which lowercases + collapses whitespace). For a + * full post body that normalization would map two structurally-different posts + * (same words, different line breaks/case) to the same id, and the second would be + * silently skipped on write — data loss of the exact gold signal SB-S1 captures. + * Byte-identity hashing makes the id true content identity: only an identical + * re-ingest collides; any real difference mints a distinct id (SB-S1). + */ +export function mintContentId(text: string): string { + return createHash("sha256").update(text).digest("hex").slice(0, 12); +} + /** * Normalize a raw provenance string to the single brain vocabulary. Trims + * lowercases, then returns the match or THROWS — provenance is load-bearing for diff --git a/scripts/brain/src/ingest.ts b/scripts/brain/src/ingest.ts new file mode 100644 index 0000000..5307e42 --- /dev/null +++ b/scripts/brain/src/ingest.ts @@ -0,0 +1,95 @@ +/** + * Ingest — the published-post gold signal (SB-S1). + * + * Captures the user's ACTUAL published posts into `ingest/published/` tagged + * `provenance=published`, so the voice/profile-learning surface can learn from + * human-published content ONLY (the model-collapse guard — never from ai-draft). + * + * This module has two halves: + * - PURE: `PublishedRecord` + `parsePublishedRecord` / `serializePublishedRecord` + * (a file-per-post constrained-header grammar; NO YAML dep, mirroring the brain + * profile.md line-grammar idiom). `parse ∘ serialize = identity` (SC2). + * - IO: `writePublished` / `ingestText` / `scanInbox` / `listPublished` — idempotent, + * collision-safe, create-on-demand under the brain `dataRoot` (SC1/SC3/SC4). + * + * The record id is `mintContentId(VERBATIM body)` (id.ts) — byte-identity, so two + * structurally-different posts never collide and the write path never silently + * drops a differing body. + */ + +import { mintContentId, normalizeProvenance } from "./id.js"; + +/** One ingested published post. `provenance` is always `published` in SB-S1. */ +export interface PublishedRecord { + /** mintContentId(body) — 12 hex, the dedupe key + filename stem. */ + id: string; + /** Always `published` — the type pins it; parse rejects any other value. */ + provenance: "published"; + /** YYYY-MM-DD — when the post was published (operator-supplied or = captured_at). */ + published_date: string; + /** YYYY-MM-DD — when this record was ingested. */ + captured_at: string; + /** `manual` (default) | a future connector token. */ + source: string; + /** The verbatim post body — byte-exact, may contain anything (`]`/`|`/newlines/`---`). */ + body: string; +} + +/** The one separator string, used identically by serialize and parse. */ +const SENTINEL = "\n---\n"; + +/** + * Serialize a record to the file grammar: a fixed 5-line header, the `---` + * sentinel, then the verbatim body. No title, no blank-line padding (the body is + * captured byte-exact, so any padding would break the round-trip identity). + */ +export function serializePublishedRecord(rec: PublishedRecord): string { + const header = [ + `id: ${rec.id}`, + `provenance: ${rec.provenance}`, + `published_date: ${rec.published_date}`, + `captured_at: ${rec.captured_at}`, + `source: ${rec.source}`, + ].join("\n"); + return header + SENTINEL + rec.body; +} + +function headerScalar(header: string, key: string): string { + const m = header.match(new RegExp(`^${key}:\\s*(.*?)\\s*$`, "m")); + if (!m) throw new Error(`malformed published record: missing "${key}:" header`); + return m[1]; +} + +/** + * Parse a record back from the file grammar. Splits on the FIRST `---` sentinel + * (the header lines are constrained `key: value` pairs that never contain one), so + * a body that itself contains `\n---\n` is preserved verbatim. The header scalars + * are read only from the header slice — a header-shaped body line cannot leak in. + * `provenance` is asserted to be `published`: a `published/` record carrying any + * other provenance is a corruption signal and throws (never silently accepted). + */ +export function parsePublishedRecord(text: string): PublishedRecord { + const idx = text.indexOf(SENTINEL); + if (idx === -1) throw new Error("malformed published record: no `---` separator"); + const header = text.slice(0, idx); + const body = text.slice(idx + SENTINEL.length); + + const id = headerScalar(header, "id"); + if (!/^[0-9a-f]{12}$/.test(id)) { + throw new Error(`malformed published record: bad id ${JSON.stringify(id)}`); + } + const provenance = normalizeProvenance(headerScalar(header, "provenance")); + if (provenance !== "published") { + throw new Error( + `corrupt published record: provenance is "${provenance}", expected "published"`, + ); + } + return { + id, + provenance, + published_date: headerScalar(header, "published_date"), + captured_at: headerScalar(header, "captured_at"), + source: headerScalar(header, "source"), + body, + }; +} diff --git a/scripts/brain/tests/ingest.test.ts b/scripts/brain/tests/ingest.test.ts new file mode 100644 index 0000000..0149067 --- /dev/null +++ b/scripts/brain/tests/ingest.test.ts @@ -0,0 +1,84 @@ +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; + +import { mintContentId } from "../src/id.js"; +import { + parsePublishedRecord, + serializePublishedRecord, + type PublishedRecord, +} from "../src/ingest.js"; + +const baseRec = (body: string): PublishedRecord => ({ + id: mintContentId(body), + provenance: "published", + published_date: "2026-05-26", + captured_at: "2026-06-23", + source: "manual", + body, +}); + +/** Round-trip helper: parse∘serialize must equal the original record. */ +function roundTrip(rec: PublishedRecord): PublishedRecord { + return parsePublishedRecord(serializePublishedRecord(rec)); +} + +describe("mintContentId — verbatim-body content hash (SC3)", () => { + test("deterministic: same body → same id", () => { + const body = "Jeg lærte noe om dømmekraft i dag.\n\nDel 1 av serien."; + assert.equal(mintContentId(body), mintContentId(body)); + }); + + test("returns a 12-hex id", () => { + assert.match(mintContentId("any body"), /^[0-9a-f]{12}$/); + }); + + test("different bodies → different ids", () => { + assert.notEqual(mintContentId("post A"), mintContentId("post B")); + }); + + test("whitespace/case differences mint DIFFERENT ids (no normalize-collision, B2 guard)", () => { + // Under specifics-bank normalizeContent these would collapse to the SAME id — + // proving the post-body hash is byte-identity, not normalized. + assert.notEqual(mintContentId("Hello World"), mintContentId("hello world")); + assert.notEqual( + mintContentId("Line one\nLine two"), + mintContentId("Line one Line two"), + ); + }); +}); + +describe("PublishedRecord grammar — parse∘serialize identity (SC2, B1 edge battery)", () => { + const bodies: Record = { + "plain multi-paragraph": "Hook line.\n\nBody paragraph one.\n\nBody paragraph two.", + "empty body": "", + "single newline": "\n", + "trailing newlines": "post text\n\n", + "body starting with ---": "---\nlooks like a separator but is body", + "body containing a \\n---\\n sentinel mid-text": "before\n---\nafter the fake separator", + "header-shaped first line": "id: 0123456789ab\nprovenance: published", + "special chars ] | quotes": 'value with ] and | and "quotes" inside', + }; + + for (const [label, body] of Object.entries(bodies)) { + test(`round-trips: ${label}`, () => { + const rec = baseRec(body); + assert.deepEqual(roundTrip(rec), rec); + }); + } + + test("serialized form has the fixed 5-line header then the --- sentinel", () => { + const text = serializePublishedRecord(baseRec("hi")); + const [header, ...rest] = text.split("\n---\n"); + assert.equal(header.split("\n").length, 5, "exactly 5 header lines before the sentinel"); + assert.equal(rest.join("\n---\n"), "hi", "body is everything after the FIRST sentinel"); + assert.match(text, /^id: [0-9a-f]{12}\nprovenance: published\n/); + }); + + test("a published/ record with non-published provenance is rejected (corruption signal)", () => { + const corrupt = serializePublishedRecord(baseRec("x")).replace( + "provenance: published", + "provenance: ai-draft", + ); + assert.throws(() => parsePublishedRecord(corrupt), /provenance/i); + }); +});