linkedin-studio/scripts/trends/src/store.ts

/**
 * Deterministic store + query for the persistent trend store (research-engine §5).
 *
 * Pure where it matters: id derivation, dedupe, topic query, and history are
 * side-effect-free and fully testable. Only loadStore/saveStore touch the
 * filesystem. No AI, no network — this module is reliable inventory, not a
 * creatively-interpreted blob. The capture that POPULATES the store (polling
 * research MCPs / web search, scoring relevance) lives in the agent/command
 * layer; this module only stores, dedupes, and serves trend signals.
 *
 * Twin of scripts/specifics-bank/src/bank.ts — same discipline, different
 * dedupe key (normalized title+URL instead of free-text content).
 */

import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs";
import { dirname, join } from "node:path";
import { homedir } from "node:os";
import { createHash } from "node:crypto";

import { SCHEMA_VERSION } from "./types.js";
import type { TrendStore, TrendRecord, TrendQueryHit } from "./types.js";

export { SCHEMA_VERSION } from "./types.js";

/** What a caller supplies to addTrend — the id is derived, never passed in. */
export interface TrendInput {
  title: string;
  url: string;
  source: string;
  capturedAt: string;
  /** The source's own publish date (ISO-8601), if known. Distinct from capturedAt; persisted first-sight, never back-filled. */
  publishedAt?: string;
  topics: string[];
  summary?: string;
}

export interface AddResult {
  store: TrendStore;
  /** true iff a new trend was appended (false = duplicate title+url). */
  added: boolean;
  /** true iff an existing duplicate gained new topic tags via union. */
  merged: boolean;
}

/** Options for a recency-ordered history slice. */
export interface HistoryOptions {
  /** Inclusive lower bound on capturedAt (ISO date); older trends are excluded. */
  since?: string;
  /** Cap on the number of returned trends (newest kept). */
  limit?: number;
}

/** Lowercase + trim + collapse all whitespace runs to a single space. */
export function normalizeField(value: string): string {
  return value.trim().toLowerCase().replace(/\s+/g, " ");
}

/**
 * Stable id = first 12 hex of sha256(normalized title + "\n" + normalized url);
 * also the dedupe key. The URL is folded to lowercase too: host case is
 * insignificant, and the combined title+url key makes a false merge on a
 * case-only path difference vanishingly unlikely. Richer URL canonicalization
 * (trailing-slash / query-param stripping) is deferred to a later slice if
 * dedup ever proves leaky.
 */
export function trendId(title: string, url: string): string {
  const key = normalizeField(title) + "\n" + normalizeField(url);
  return createHash("sha256").update(key).digest("hex").slice(0, 12);
}

export function emptyStore(): TrendStore {
  return { schemaVersion: SCHEMA_VERSION, trends: [] };
}

/** Read the store; a missing file is an empty store (never throws on absence). */
export function loadStore(path: string): TrendStore {
  if (!existsSync(path)) return emptyStore();
  const parsed = JSON.parse(readFileSync(path, "utf8")) as Partial<TrendStore>;
  // Forward migrate-on-load: stamp to the current version, never downgrade. v1→v2 is
  // purely additive-optional (an old record is already a valid v2 record that simply
  // lacks the optional publishedAt), so the migration is the version stamp alone —
  // records pass through untouched (lossless + idempotent for any well-formed store).
  // A string / NaN / absent version coerces to the current version (never crashes); the
  // non-array `trends` coercion below is unchanged and out of the losslessness claim.
  const onDisk = typeof parsed.schemaVersion === "number" ? parsed.schemaVersion : SCHEMA_VERSION;
  return {
    schemaVersion: Math.max(onDisk, SCHEMA_VERSION),
    trends: Array.isArray(parsed.trends) ? parsed.trends : [],
  };
}

/** Write the store as pretty JSON, creating the parent dir if needed. */
export function saveStore(path: string, store: TrendStore): void {
  mkdirSync(dirname(path), { recursive: true });
  writeFileSync(path, JSON.stringify(store, null, 2) + "\n", "utf8");
}

/** Union of two tag lists, order-stable on the first list, case-insensitive dedupe. */
function unionTopics(existing: string[], incoming: string[]): { topics: string[]; changed: boolean } {
  const seen = new Set(existing.map((t) => t.toLowerCase()));
  const topics = [...existing];
  let changed = false;
  for (const t of incoming) {
    if (!seen.has(t.toLowerCase())) {
      seen.add(t.toLowerCase());
      topics.push(t);
      changed = true;
    }
  }
  return { topics, changed };
}

/**
 * Add a trend, deduping on normalized title+url. A duplicate does not append a
 * second entry — instead its topic tags are unioned in, so the same trend
 * re-surfaced under a new edition's topics enriches the existing record. The
 * FIRST sighting's source + capturedAt are kept (provenance of first sight);
 * title/url/summary are stored VERBATIM, only the id is normalized.
 */
export function addTrend(store: TrendStore, input: TrendInput): AddResult {
  const id = trendId(input.title, input.url);
  const existing = store.trends.find((t) => t.id === id);
  if (existing) {
    const { topics, changed } = unionTopics(existing.topics, input.topics);
    existing.topics = topics;
    return { store, added: false, merged: changed };
  }
  const trend: TrendRecord = {
    id,
    title: input.title,
    url: input.url,
    source: input.source,
    capturedAt: input.capturedAt,
    ...(input.publishedAt !== undefined ? { publishedAt: input.publishedAt } : {}),
    topics: [...input.topics],
    ...(input.summary !== undefined ? { summary: input.summary } : {}),
  };
  store.trends.push(trend);
  return { store, added: true, merged: false };
}

/**
 * Trends whose topics overlap the query, ranked by overlap (desc) then recency
 * (capturedAt desc). Topic matching is case-insensitive. Non-matches are
 * excluded.
 */
export function queryByTopic(store: TrendStore, topics: string[]): TrendQueryHit[] {
  const wanted = topics.map((t) => t.toLowerCase());
  const hits: TrendQueryHit[] = [];
  for (const trend of store.trends) {
    const have = new Set(trend.topics.map((t) => t.toLowerCase()));
    const topicOverlap = wanted.reduce((n, t) => (have.has(t) ? n + 1 : n), 0);
    if (topicOverlap > 0) hits.push({ trend, topicOverlap });
  }
  hits.sort(
    (a, b) => b.topicOverlap - a.topicOverlap || b.trend.capturedAt.localeCompare(a.trend.capturedAt),
  );
  return hits;
}

/**
 * Trend history, newest first. Optionally filtered to capturedAt >= `since`
 * (inclusive) and capped to `limit`. The time-scoped complement to
 * queryByTopic's topic-scoped view; together they are the "historikk-query".
 */
export function history(store: TrendStore, opts: HistoryOptions = {}): TrendRecord[] {
  let out = [...store.trends];
  if (opts.since !== undefined) out = out.filter((t) => t.capturedAt >= opts.since!);
  out.sort((a, b) => b.capturedAt.localeCompare(a.capturedAt));
  if (opts.limit !== undefined) out = out.slice(0, opts.limit);
  return out;
}

/**
 * The most recent capturedAt across the store (ISO date), or null when the
 * store is empty. The staleness signal behind the SessionStart trend-freshness
 * nudge (B-S3) and the `status` CLI subcommand: reuses history()'s newest-first
 * ordering so the module has a single notion of "newest". Pure — the days-stale
 * arithmetic (needs "today") lives in the caller, not here.
 */
export function newestCaptureDate(store: TrendStore): string | null {
  return history(store, { limit: 1 })[0]?.capturedAt ?? null;
}

/**
 * Default store path under the per-user data dir (M0 data-path convention), so
 * the trend history survives plugin upgrades/reinstalls. `LINKEDIN_STUDIO_DATA`
 * overrides the root; otherwise `~/.claude/linkedin-studio`.
 */
export function defaultStorePath(): string {
  const root = process.env.LINKEDIN_STUDIO_DATA ?? join(homedir(), ".claude", "linkedin-studio");
  return join(root, "trends", "trends.json");
}