From ff39d14206d4fae1c83345e8c93f1a3e75567241 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 17:00:56 +0200 Subject: [PATCH] =?UTF-8?q?feat(linkedin-studio):=20SB-S2=20consolidation?= =?UTF-8?q?=20engine=20=E2=80=94=20proposeDiff/applyDiff=20[skip-docs]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure deterministic engine over the two-layer profile: add / reject-ai-draft / evidence-bump / promote-at-N(3) / conflict-keep-both / decay-flag(90d). No-duplicate-id guarantee: primary id = mintEntityId(observed,key), conflict-alt id = mintContentId(observed-alt:key::value::date) — byte-distinct. Folded profile-field seeds immutable (different kind); static facts decay-exempt; no supersede (S3). applyDiff produces the next ProfileDoc that round-trips through parse/serialize; re-running is idempotent (bump, not duplicate). + consolidation-state.json sidecar IO. 12 engine tests (SC1a–g, SC2, SC3, SC4). brain 63→75, tsc clean. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01RigJBiRFNtFZKCz21qNbQ4 --- scripts/brain/src/consolidate.ts | 186 ++++++++++++++++++++++++ scripts/brain/tests/consolidate.test.ts | 160 ++++++++++++++++++++ 2 files changed, 346 insertions(+) create mode 100644 scripts/brain/src/consolidate.ts create mode 100644 scripts/brain/tests/consolidate.test.ts diff --git a/scripts/brain/src/consolidate.ts b/scripts/brain/src/consolidate.ts new file mode 100644 index 0000000..65372f8 --- /dev/null +++ b/scripts/brain/src/consolidate.ts @@ -0,0 +1,186 @@ +/** + * Consolidation engine (SB-S2) — the compounding mechanism. + * + * PURE core: `proposeDiff` reads candidate facts + the current two-layer profile + * and returns a typed DIFF (never mutates, never writes); `applyDiff` produces the + * next ProfileDoc. The operator-gated CLI is the only thing that writes profile.md. + * + * Invariants enforced IN CODE (not just docs): + * - provenance-gated: `ai-draft` candidates are rejected outright (model-collapse guard); + * - evidence-threshold promotion (dynamic→static at N observations); + * - contradiction → keep-both with DISTINCT ids (no supersede in S2 — that's S3); + * - temporal decay flagging (dynamic facts only; static facts are decay-exempt). + * + * Id model (the no-duplicate-id guarantee): a concept's PRIMARY fact id is + * `mintEntityId({kind:'observed', key})` (key-only); a conflict ALT fact id is + * `mintContentId('observed-alt:'+key+'::'+value+'::'+date)` (byte-distinct). Folded + * `profile-field` static seeds use a different kind, so consolidation never collides + * with them (they stay immutable in S2). Matching always starts from the candidate's + * key, so an existing fact's key never needs to be recovered. + */ + +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; + +import { dataRoot } from "./dataRoot.js"; +import { mintContentId, mintEntityId } from "./id.js"; +import { SCHEMA_VERSION } from "./types.js"; +import type { ProfileDoc, ProfileFact, Provenance } from "./types.js"; + +const OBSERVED_KIND = "observed"; + +/** A candidate fact the invoking session extracts from the gathered deltas. */ +export interface Candidate { + key: string; // the concept (keyed for the id; single-line) + value: string; // single-line (no newline/CR — profile grammar) + provenance: Provenance; + source: string; // e.g. "published:" | "manual" + observed_date: string; // YYYY-MM-DD +} + +export interface ProfileDiff { + additions: ProfileFact[]; // new dynamic facts (primary adds + conflict alts) + evidenceBumps: { id: string; newCount: number; last_seen: string }[]; + promotions: { id: string }[]; // dynamic→static (post-bump count ≥ N) + conflicts: { primaryId: string; primaryValue: string; altId: string }[]; + staleFlags: { id: string; last_seen: string; daysStale: number }[]; +} + +export interface ConsolidateOpts { + promoteThreshold?: number; // default 3 + decayDays?: number; // default 90 +} + +function daysBetween(from: string, to: string): number { + return Math.floor((Date.parse(to) - Date.parse(from)) / 86400000); +} + +function altId(c: Candidate): string { + return mintContentId(`observed-alt:${c.key}::${c.value}::${c.observed_date}`); +} + +function newFact(id: string, c: Candidate, today: string): ProfileFact { + return { + id, + value: c.value, + first_seen: c.observed_date, + last_seen: today, + evidence_count: 1, + provenance: c.provenance, + status: "active", + }; +} + +/** + * Propose a diff over the current profile from a batch of candidates. Pure — never + * mutates `current`/`candidates`, never touches the filesystem. + */ +export function proposeDiff(args: { + current: ProfileDoc; + candidates: Candidate[]; + today: string; + opts?: ConsolidateOpts; +}): ProfileDiff { + const { current, candidates, today, opts } = args; + const N = opts?.promoteThreshold ?? 3; + const DECAY = opts?.decayDays ?? 90; + + const byId = new Map([...current.static, ...current.dynamic].map((f) => [f.id, f])); + const dynamicIds = new Set(current.dynamic.map((f) => f.id)); + + const additions: ProfileFact[] = []; + const evidenceBumps: ProfileDiff["evidenceBumps"] = []; + const promotions: ProfileDiff["promotions"] = []; + const conflicts: ProfileDiff["conflicts"] = []; + + // Track ids added/bumped this pass so a repeated candidate in one batch doesn't double-add. + const touched = new Set(); + const bump = (id: string, prevCount: number) => { + const newCount = prevCount + 1; + evidenceBumps.push({ id, newCount, last_seen: today }); + if (dynamicIds.has(id) && newCount >= N) promotions.push({ id }); + }; + + for (const c of candidates) { + if (c.provenance === "ai-draft") continue; // model-collapse guard (SC1b) + const primaryId = mintEntityId({ kind: OBSERVED_KIND, key: c.key }); + const prev = byId.get(primaryId); + + if (!prev) { + if (!touched.has(primaryId)) { + additions.push(newFact(primaryId, c, today)); + touched.add(primaryId); + } + continue; + } + if (prev.value === c.value) { + bump(primaryId, prev.evidence_count); // SC1c/SC1d + continue; + } + // conflict — keep both with a distinct alt id (SC1e), old fact untouched + const aId = altId(c); + const altPrev = byId.get(aId); + if (altPrev && altPrev.value === c.value) { + bump(aId, altPrev.evidence_count); // idempotent re-conflict → bump the alt + } else if (!touched.has(aId)) { + additions.push(newFact(aId, c, today)); + conflicts.push({ primaryId, primaryValue: prev.value, altId: aId }); + touched.add(aId); + } + } + + const staleFlags = current.dynamic + .filter((f) => daysBetween(f.last_seen, today) > DECAY) + .map((f) => ({ id: f.id, last_seen: f.last_seen, daysStale: daysBetween(f.last_seen, today) })); + + return { additions, evidenceBumps, promotions, conflicts, staleFlags }; +} + +/** + * Apply a proposed diff to produce the next ProfileDoc. Pure (returns a new doc). + * Because primary and alt ids are byte-distinct, no two facts ever share an id, so + * the bump/promote targets are unambiguous and the doc stays well-formed (SC3). + */ +export function applyDiff(current: ProfileDoc, diff: ProfileDiff): ProfileDoc { + const bumpMap = new Map(diff.evidenceBumps.map((b) => [b.id, b])); + const applyBump = (f: ProfileFact): ProfileFact => { + const b = bumpMap.get(f.id); + return b ? { ...f, evidence_count: b.newCount, last_seen: b.last_seen } : { ...f }; + }; + + let staticF = current.static.map(applyBump); + let dynamicF = current.dynamic.map(applyBump); + + const promoteIds = new Set(diff.promotions.map((p) => p.id)); + const promoted = dynamicF.filter((f) => promoteIds.has(f.id)); + dynamicF = dynamicF.filter((f) => !promoteIds.has(f.id)); + staticF = [...staticF, ...promoted]; + + dynamicF = [...dynamicF, ...diff.additions.map((f) => ({ ...f }))]; + + return { schemaVersion: SCHEMA_VERSION, static: staticF, dynamic: dynamicF }; +} + +// ── consolidation-state sidecar (the only IO; brain data-root, reachable by both +// the CLI via dataRoot and the session-start hook via getDataRoot) ──────────── + +const STATE_SUB = "brain/consolidation-state.json"; + +/** Read the last-run date; absent/malformed → {last_run:null}. */ +export function readConsolidationState(): { last_run: string | null } { + const p = dataRoot(STATE_SUB); + if (!existsSync(p)) return { last_run: null }; + try { + const j = JSON.parse(readFileSync(p, "utf8")); + return { last_run: typeof j?.last_run === "string" ? j.last_run : null }; + } catch { + return { last_run: null }; + } +} + +/** Record the last-run date (called only by the gated --apply path). */ +export function writeConsolidationState(date: string): void { + const p = dataRoot(STATE_SUB); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, JSON.stringify({ last_run: date }, null, 2) + "\n", "utf8"); +} diff --git a/scripts/brain/tests/consolidate.test.ts b/scripts/brain/tests/consolidate.test.ts new file mode 100644 index 0000000..52eb11b --- /dev/null +++ b/scripts/brain/tests/consolidate.test.ts @@ -0,0 +1,160 @@ +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; + +import { proposeDiff, applyDiff, type Candidate } from "../src/consolidate.js"; +import { mintEntityId } from "../src/id.js"; +import { parseProfile, serializeProfile } from "../src/profile.js"; +import type { ProfileDoc, ProfileFact } from "../src/types.js"; + +const TODAY = "2026-06-23"; +const observedId = (key: string) => mintEntityId({ kind: "observed", key }); + +const fact = (over: Partial): ProfileFact => ({ + id: "000000000000", + value: "v", + first_seen: "2026-01-01", + last_seen: TODAY, + evidence_count: 1, + provenance: "published", + status: "active", + ...over, +}); +const doc = (staticF: ProfileFact[], dynamicF: ProfileFact[]): ProfileDoc => ({ + schemaVersion: 1, + static: staticF, + dynamic: dynamicF, +}); +const cand = (over: Partial): Candidate => ({ + key: "k", + value: "v", + provenance: "published", + source: "manual", + observed_date: TODAY, + ...over, +}); + +describe("proposeDiff classification (SC1a–g)", () => { + test("SC1a — add: a published candidate with no match becomes a new dynamic fact", () => { + const diff = proposeDiff({ current: doc([], []), candidates: [cand({ key: "topic", value: "AI safety" })], today: TODAY }); + assert.equal(diff.additions.length, 1); + assert.equal(diff.additions[0].value, "AI safety"); + assert.equal(diff.additions[0].id, observedId("topic")); + assert.equal(diff.additions[0].evidence_count, 1); + assert.equal(diff.additions[0].status, "active"); + }); + + test("SC1b — reject: an ai-draft candidate produces an empty diff", () => { + const diff = proposeDiff({ current: doc([], []), candidates: [cand({ provenance: "ai-draft" })], today: TODAY }); + assert.deepEqual(diff.additions, []); + assert.deepEqual(diff.evidenceBumps, []); + assert.deepEqual(diff.promotions, []); + assert.deepEqual(diff.conflicts, []); + }); + + test("SC1c — evidence-bump: same key+value bumps the existing fact, no addition", () => { + const id = observedId("topic"); + const cur = doc([], [fact({ id, value: "AI safety", evidence_count: 1 })]); + const diff = proposeDiff({ current: cur, candidates: [cand({ key: "topic", value: "AI safety" })], today: TODAY }); + assert.equal(diff.additions.length, 0); + assert.equal(diff.evidenceBumps.length, 1); + assert.deepEqual(diff.evidenceBumps[0], { id, newCount: 2, last_seen: TODAY }); + }); + + test("SC1d — promote: a dynamic fact reaching N=3 is promoted", () => { + const id = observedId("topic"); + const cur = doc([], [fact({ id, value: "AI safety", evidence_count: 2 })]); + const diff = proposeDiff({ current: cur, candidates: [cand({ key: "topic", value: "AI safety" })], today: TODAY }); + assert.equal(diff.evidenceBumps[0].newCount, 3); + assert.deepEqual(diff.promotions, [{ id }]); + }); + + test("SC1d — no promote below N", () => { + const id = observedId("topic"); + const cur = doc([], [fact({ id, value: "x", evidence_count: 1 })]); + const diff = proposeDiff({ current: cur, candidates: [cand({ key: "topic", value: "x" })], today: TODAY }); + assert.deepEqual(diff.promotions, []); + }); + + test("SC1e — conflict: different value keeps BOTH with DISTINCT ids, old fact untouched", () => { + const primaryId = observedId("role"); + const cur = doc([], [fact({ id: primaryId, value: "advisor", evidence_count: 2 })]); + const diff = proposeDiff({ current: cur, candidates: [cand({ key: "role", value: "architect" })], today: TODAY }); + assert.equal(diff.conflicts.length, 1); + assert.equal(diff.conflicts[0].primaryId, primaryId); + assert.equal(diff.conflicts[0].primaryValue, "advisor"); + assert.notEqual(diff.conflicts[0].altId, primaryId, "alt id must differ from primary id (no duplicate-id corruption)"); + // the alt fact is added; the old fact is NOT bumped + assert.equal(diff.additions.length, 1); + assert.equal(diff.additions[0].value, "architect"); + assert.equal(diff.additions[0].id, diff.conflicts[0].altId); + assert.deepEqual(diff.evidenceBumps, []); + }); + + test("SC1f — decay: a dynamic fact older than 90d is flagged; a static fact is decay-exempt", () => { + const old = "2026-03-01"; // ~114 days before TODAY + const dyn = fact({ id: observedId("stale-topic"), value: "old", last_seen: old }); + const stat = fact({ id: observedId("settled"), value: "stable", last_seen: old }); + const cur = doc([stat], [dyn]); + const diff = proposeDiff({ current: cur, candidates: [], today: TODAY }); + assert.equal(diff.staleFlags.length, 1, "only the dynamic stale fact is flagged"); + assert.equal(diff.staleFlags[0].id, dyn.id); + assert.ok(diff.staleFlags[0].daysStale > 90); + }); + + test("SC1f — a fresh dynamic fact is not flagged", () => { + const cur = doc([], [fact({ id: observedId("fresh"), last_seen: "2026-06-20" })]); + const diff = proposeDiff({ current: cur, candidates: [], today: TODAY }); + assert.deepEqual(diff.staleFlags, []); + }); + + test("SC1g — folded immutable: a candidate overlapping a profile-field seed adds a new observed fact, seed untouched", () => { + const foldedId = mintEntityId({ kind: "profile-field", key: "role" }); + const folded = fact({ id: foldedId, value: "advisor", provenance: "human" }); + const cur = doc([folded], []); + const before = serializeProfile(cur); + const diff = proposeDiff({ current: cur, candidates: [cand({ key: "role", value: "architect" })], today: TODAY }); + // candidate keys to observed:role (≠ profile-field:role) → no match → ADD, no conflict on the folded seed + assert.equal(diff.additions.length, 1); + assert.equal(diff.additions[0].id, observedId("role")); + assert.equal(diff.conflicts.length, 0); + assert.equal(serializeProfile(cur), before, "input doc not mutated; folded seed untouched"); + }); +}); + +describe("proposeDiff purity + applyDiff round-trip (SC2, SC3, SC4)", () => { + test("SC2 — proposeDiff does not mutate its inputs", () => { + const cur = doc([], [fact({ id: observedId("topic"), value: "x" })]); + const candidates = [cand({ key: "topic", value: "x" }), cand({ key: "new", value: "y" })]; + const snapCur = JSON.stringify(cur); + const snapCand = JSON.stringify(candidates); + proposeDiff({ current: cur, candidates, today: TODAY }); + assert.equal(JSON.stringify(cur), snapCur, "current unchanged"); + assert.equal(JSON.stringify(candidates), snapCand, "candidates unchanged"); + }); + + test("SC3 — applyDiff(current, proposeDiff(...)) round-trips through parse/serialize", () => { + const cur = doc([fact({ id: mintEntityId({ kind: "profile-field", key: "name" }), value: "KTG", provenance: "human" })], [fact({ id: observedId("topic"), value: "AI safety", evidence_count: 2 })]); + const candidates = [ + cand({ key: "topic", value: "AI safety" }), // bump→promote + cand({ key: "new-topic", value: "governance" }), // add + cand({ key: "topic", value: "alignment" }), // conflict on topic → alt + ]; + const diff = proposeDiff({ current: cur, candidates, today: TODAY }); + const next = applyDiff(cur, diff); + const round = parseProfile(serializeProfile(next)); + assert.deepEqual(round, next, "applied doc round-trips exactly"); + // every fact id is unique (no duplicate-id corruption) + const ids = [...round.static, ...round.dynamic].map((f) => f.id); + assert.equal(new Set(ids).size, ids.length, "all fact ids unique"); + }); + + test("SC4 — idempotent: re-running propose→apply with the same candidates adds no duplicate facts", () => { + const cur = doc([], []); + const candidates = [cand({ key: "topic", value: "AI safety" })]; + const once = applyDiff(cur, proposeDiff({ current: cur, candidates, today: TODAY })); + const twice = applyDiff(once, proposeDiff({ current: once, candidates, today: TODAY })); + const factsOnce = [...once.static, ...once.dynamic].length; + const factsTwice = [...twice.static, ...twice.dynamic].length; + assert.equal(factsTwice, factsOnce, "no duplicate fact on re-run (bump only)"); + }); +});