linkedin-studio/scripts/brain/src/cli.ts

#!/usr/bin/env node
/**
 * CLI for the second-brain foundation + ingest (SB-S0 / SB-S1).
 *
 *   node --import tsx src/cli.ts init
 *   node --import tsx src/cli.ts ingest --file <path> [--source <s>] [--date <YYYY-MM-DD>]
 *   node --import tsx src/cli.ts ingest --scan-inbox [--source <s>]
 *   node --import tsx src/cli.ts published list [--json]
 *
 * `init` scaffolds the brain/ + ingest/ tree (idempotent). `ingest` captures the
 * user's published posts into ingest/published/ tagged provenance=published — the
 * gold signal the voice/profile-learning surface learns from (never ai-draft).
 *
 * Exit code: 0 on success, 2 on usage error.
 */

import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";

import {
  applyDiff,
  proposeDiff,
  readConsolidationState,
  writeConsolidationState,
  type Candidate,
  type ProfileDiff,
} from "./consolidate.js";
import { assemblePostGraph, loadAnalyticsRows } from "./assemble.js";
import { loadRecentPosts, reconcileRecentPosts, summarizeReconcile } from "./reconcile.js";
import { dataRoot } from "./dataRoot.js";
import { ingestText, listPublished, parsePublishedRecord, scanInbox } from "./ingest.js";
import { parseProfile, serializeProfile } from "./profile.js";
import { initBrain } from "./scaffold.js";
import { SCHEMA_VERSION } from "./types.js";
import type { ProfileDoc } from "./types.js";

const PROVENANCES = ["human", "published", "ai-draft"];
const EMPTY_PROFILE: ProfileDoc = { schemaVersion: SCHEMA_VERSION, static: [], dynamic: [] };

function loadProfile(): ProfileDoc {
  const p = dataRoot(join("brain", "profile.md"));
  return existsSync(p) ? parseProfile(readFileSync(p, "utf8")) : EMPTY_PROFILE;
}

/** Parse `--key value` / boolean `--flag` args (the specifics-bank CLI idiom). */
function parseFlags(args: string[]): Record<string, string> {
  const out: Record<string, string> = {};
  for (let i = 0; i < args.length; i++) {
    const a = args[i];
    if (a.startsWith("--")) {
      const key = a.slice(2);
      const next = args[i + 1];
      if (next === undefined || next.startsWith("--")) {
        out[key] = "true";
      } else {
        out[key] = next;
        i++;
      }
    }
  }
  return out;
}

/**
 * SB-S3c: collect EVERY value of a repeatable `--key <value>` flag into an array
 * (the single-value `parseFlags` keeps only the last). Scans the raw args directly,
 * leaving `parseFlags` untouched — so single-value flag behaviour is unchanged.
 */
function collectRepeated(args: string[], key: string): string[] {
  const out: string[] = [];
  for (let i = 0; i < args.length; i++) {
    if (args[i] === `--${key}`) {
      const next = args[i + 1];
      if (next !== undefined && !next.startsWith("--")) {
        out.push(next);
        i++;
      }
    }
  }
  return out;
}

function today(): string {
  return new Date().toISOString().slice(0, 10);
}

function usage(msg: string): never {
  console.error(`error: ${msg}`);
  console.error(
    "usage:\n" +
      "  init\n" +
      "  ingest --file <path> [--source <s>] [--date <YYYY-MM-DD>] [--specific <id>]… [--trend <id>]…\n" +
      "  ingest --scan-inbox [--source <s>]\n" +
      "  published list [--json]\n" +
      "  assemble\n" +
      "  reconcile",
  );
  process.exit(2);
}

function runInit(): void {
  const { created, skipped } = initBrain();
  console.log(`Brain root: ${dataRoot("brain")}`);
  if (created.length > 0) {
    console.log(`Created (${created.length}):`);
    for (const c of created) console.log(`  + ${c}`);
  }
  if (skipped.length > 0) {
    console.log(`Skipped — already present (${skipped.length}):`);
    for (const s of skipped) console.log(`  · ${s}`);
  }
  if (created.length === 0) console.log("Already initialised — nothing to do.");
}

function runIngest(rest: string[], flags: Record<string, string>): void {
  if (flags["scan-inbox"] === "true") {
    const res = scanInbox({ captured_at: today(), source: flags.source });
    console.log(
      `Scanned inbox → published: ${res.processed.length} new, ${res.skipped.length} already published`,
    );
    return;
  }
  const file = flags.file;
  if (!file || file === "true") usage("ingest needs --file <path> or --scan-inbox");
  const body = readFileSync(file, "utf8");
  // SB-S3c: --specific / --trend are repeatable — read ONLY via collectRepeated.
  const res = ingestText({
    body,
    captured_at: today(),
    source: flags.source,
    published_date: flags.date === "true" ? undefined : flags.date,
    specifics: collectRepeated(rest, "specific"),
    trends: collectRepeated(rest, "trend"),
  });
  if (res.written && res.collision) {
    console.log(`Collision (different body, same id) → wrote ${res.path}`);
  } else if (res.written) {
    console.log(`Wrote ingest/published/${res.record.id}.md`);
  } else {
    console.log(`Duplicate — already published: ${res.record.id}`);
  }
}

function runPublished(rest: string[], flags: Record<string, string>): void {
  if (rest[0] !== "list") usage("published needs: list");
  const { records, skipped } = listPublished();
  if (flags.json === "true") {
    console.log(JSON.stringify({ records, skipped }, null, 2));
    return;
  }
  const tail = skipped > 0 ? ` (${skipped} unreadable skipped)` : "";
  console.log(`Published gold corpus: ${records.length} record(s)${tail}`);
  for (const r of records) {
    console.log(`  · ${r.id} · ${r.provenance} · ${r.published_date} · ${r.firstLine}`);
  }
}

/**
 * SB-S3c: read-only cross-silo assembler view — post → raw-material → performance.
 * Loads published records + analytics rows (inlined raw-JSON, no analytics import),
 * prints the join newest-first. Writes NOTHING.
 */
function runAssemble(_flags: Record<string, string>): void {
  const pubDir = dataRoot(join("ingest", "published"));
  const records = existsSync(pubDir)
    ? readdirSync(pubDir)
        .filter((f) => f.endsWith(".md") && !f.startsWith("."))
        .map((f) => {
          try {
            return parsePublishedRecord(readFileSync(join(pubDir, f), "utf8"));
          } catch {
            return null;
          }
        })
        .filter((r): r is NonNullable<typeof r> => r !== null)
    : [];
  const analytics = loadAnalyticsRows();
  const bodyById = new Map(records.map((r) => [r.id, r.body]));
  const graph = assemblePostGraph({ records, analytics }).sort((a, b) =>
    b.published_date.localeCompare(a.published_date),
  );
  if (graph.length === 0) {
    console.log(
      "No published records to assemble. Ingest posts with `brain ingest --file <p> [--specific <id>] [--trend <id>]`.",
    );
    return;
  }
  console.log(`Post graph — ${graph.length} record(s); ${analytics.length} analytics row(s):`);
  for (const node of graph) {
    const firstLine = (bodyById.get(node.contentId) ?? "").split("\n", 1)[0];
    console.log(`\n· ${node.contentId} · ${node.published_date} · ${firstLine}`);
    console.log(`  specifics: ${node.specifics.length ? node.specifics.join(", ") : "—"}`);
    console.log(`  trends: ${node.trends.length ? node.trends.join(", ") : "—"}`);
    if (node.match.confidence === "none") {
      console.log("  analytics: none (no title-prefix+date match)");
    } else {
      const eng = node.match.row?.metrics?.engagementRate;
      console.log(
        `  analytics: ${node.match.confidence}${eng !== undefined ? ` [eng ${eng}%]` : ""}`,
      );
    }
  }
}

/**
 * SB-S3e: read-only triple-post reconciliation — silo 1 (`## Recent Posts`,
 * auto-tracked creation) vs the silo 2↔3 graph. Surfaces the coverage gap
 * (created posts never `brain ingest`-ed). Writes NOTHING. Loads real
 * `PublishedRecord[]` (with `body`) via the same inline parse as `runAssemble`
 * — NOT `listPublished` (body-less). Calls the core by its literal name.
 */
function runReconcile(_flags: Record<string, string>): void {
  const pubDir = dataRoot(join("ingest", "published"));
  const records = existsSync(pubDir)
    ? readdirSync(pubDir)
        .filter((f) => f.endsWith(".md") && !f.startsWith("."))
        .map((f) => {
          try {
            return parsePublishedRecord(readFileSync(join(pubDir, f), "utf8"));
          } catch {
            return null;
          }
        })
        .filter((r): r is NonNullable<typeof r> => r !== null)
    : [];
  const analytics = loadAnalyticsRows();
  const graph = assemblePostGraph({ records, analytics });
  const recentPosts = loadRecentPosts();
  const nodes = reconcileRecentPosts({ recentPosts, records, graph });
  const sum = summarizeReconcile(nodes);
  if (nodes.length === 0) {
    console.log(
      "No tracked posts in `## Recent Posts` to reconcile (check that posts were created via the plugin, or that STATE_FILE points at your state file).",
    );
    return;
  }
  console.log(
    `Reconcile — ${sum.created} created · ${sum.inGraph} in-graph · ${sum.inBrainOnly} in-brain-only · ${sum.orphaned} orphaned:`,
  );
  for (const n of nodes) {
    const id = n.contentId ? ` · ${n.contentId}` : "";
    console.log(`\n· ${n.recentPost.date} · ${n.tier}${id}\n  "${n.recentPost.hook}"`);
  }
  if (sum.orphaned > 0) {
    console.log(
      `\n${sum.orphaned} created post(s) are not in the brain graph — \`brain ingest --file <p> [--specific <id>] [--trend <id>]\` to feed them.`,
    );
  }
}

function renderDiffMd(diff: ProfileDiff): string {
  const lines = ["---", "type: PendingDiff", "---", "", "# Pending profile diff", "", "> Operator-gated. Review, then `brain consolidate --apply --diff brain/pending-diff.json --confirm`.", ""];
  const section = (title: string, items: string[]) => {
    lines.push(`## ${title} (${items.length})`, "");
    for (const i of items) lines.push(`- ${i}`);
    lines.push("");
  };
  section("Additions", diff.additions.map((f) => `${f.value}  [${f.provenance}, id ${f.id}]`));
  section("Evidence-bumps", diff.evidenceBumps.map((b) => `${b.id} → count ${b.newCount}`));
  section("Promotions (dynamic→static)", diff.promotions.map((p) => p.id));
  section("⚠ Conflicts (both kept)", diff.conflicts.map((c) => `"${c.primaryValue}" (${c.primaryId}) vs new (${c.altId})`));
  section("Stale (dynamic, >decay)", diff.staleFlags.map((s) => `${s.id} — last seen ${s.last_seen} (${s.daysStale}d)`));
  // SB-S3b: rendered LAST and only when present, so a zero-supersession diff stays byte-identical.
  if ((diff.supersedes ?? []).length > 0) {
    section("Supersessions (old → new)", diff.supersedes.map((s) => `\`${s.oldValue}\` → \`${s.winner.value}\` [${s.winner.provenance}, retired ${s.oldId}]`));
  }
  return lines.join("\n") + "\n";
}

function validateCandidates(raw: unknown): Candidate[] {
  if (!Array.isArray(raw)) usage("candidates file must be a JSON array");
  (raw as unknown[]).forEach((c: any, i) => {
    for (const k of ["key", "value", "source", "observed_date"]) {
      if (typeof c?.[k] !== "string" || c[k] === "") usage(`candidate ${i}: missing/empty "${k}"`);
    }
    if (!PROVENANCES.includes(c.provenance)) usage(`candidate ${i}: provenance must be one of ${PROVENANCES.join(", ")}`);
    if (/[\n\r]/.test(c.key) || /[\n\r]/.test(c.value)) usage(`candidate ${i}: key/value must be single-line (no newline/CR)`);
    // SB-S3b: the optional supersede signal, when present, is a non-empty single-line target key.
    if (c.supersedes !== undefined && (typeof c.supersedes !== "string" || c.supersedes === "" || /[\n\r]/.test(c.supersedes))) {
      usage(`candidate ${i}: "supersedes" must be a non-empty single-line string when present`);
    }
  });
  return raw as Candidate[];
}

function runConsolidate(flags: Record<string, string>): void {
  if (flags.gather === "true") {
    const { last_run } = readConsolidationState();
    const pubDir = dataRoot(join("ingest", "published"));
    const records = existsSync(pubDir)
      ? readdirSync(pubDir)
          .filter((f) => f.endsWith(".md") && !f.startsWith("."))
          .map((f) => {
            try { return parsePublishedRecord(readFileSync(join(pubDir, f), "utf8")); } catch { return null; }
          })
          .filter((r): r is NonNullable<typeof r> => r !== null)
      : [];
    const fresh = records.filter((r) => last_run === null || r.published_date > last_run);
    const profile = loadProfile();
    if (flags.json === "true") {
      // SB-S3b: only ACTIVE facts are live context — superseded archival facts must not be re-presented to the extraction session.
      console.log(JSON.stringify({ since: last_run, published: fresh.map((r) => ({ id: r.id, published_date: r.published_date, body: r.body })), profileFacts: [...profile.static, ...profile.dynamic].filter((f) => f.status === "active") }, null, 2));
      return;
    }
    console.log(`Consolidation gather — ${fresh.length} new published record(s) since ${last_run ?? "the beginning"}:`);
    for (const r of fresh) {
      console.log(`\n## ${r.id} (${r.published_date})`);
      console.log(r.body);
    }
    console.log(`\nCurrent profile: ${profile.static.length} static + ${profile.dynamic.length} dynamic fact(s).`);
    console.log(`\n→ Extract a Candidate[] from the above, then: brain consolidate --propose --candidates <file.json>`);
    return;
  }

  if (flags.propose === "true") {
    const cf = flags.candidates;
    if (!cf || cf === "true") usage("--propose needs --candidates <file.json>");
    let raw: unknown;
    try { raw = JSON.parse(readFileSync(cf, "utf8")); } catch { usage("candidates file is not valid JSON"); }
    const candidates = validateCandidates(raw);
    const diff = proposeDiff({ current: loadProfile(), candidates, today: today() });
    mkdirSync(dataRoot("brain"), { recursive: true });
    const jsonPath = dataRoot(join("brain", "pending-diff.json"));
    const mdPath = dataRoot(join("brain", "pending-diff.md"));
    writeFileSync(jsonPath, JSON.stringify(diff, null, 2) + "\n", "utf8");
    writeFileSync(mdPath, renderDiffMd(diff), "utf8");
    console.log(`Wrote ${jsonPath}\nWrote ${mdPath}`);
    console.log(`Review ${mdPath}, then: brain consolidate --apply --diff ${jsonPath} --confirm`);
    return;
  }

  if (flags.apply === "true") {
    const df = flags.diff;
    if (!df || df === "true") usage("--apply needs --diff <file.json>");
    if (flags.confirm !== "true") {
      console.error("error: refusing to apply without --confirm (operator gate)");
      process.exit(1);
    }
    const diff = JSON.parse(readFileSync(df, "utf8")) as ProfileDiff;
    const next = applyDiff(loadProfile(), diff);
    const profilePath = dataRoot(join("brain", "profile.md"));
    mkdirSync(dirname(profilePath), { recursive: true });
    writeFileSync(profilePath, serializeProfile(next), "utf8");
    writeConsolidationState(today());
    console.log(`Applied diff to ${profilePath}; consolidation recorded (last_run = ${today()}).`);
    return;
  }

  usage("consolidate needs --gather | --propose --candidates <f> | --apply --diff <f> --confirm");
}

function main(): void {
  const [command, ...rest] = process.argv.slice(2);
  const flags = parseFlags(rest);

  if (command === "init") return runInit();
  if (command === "ingest") return runIngest(rest, flags);
  if (command === "published") return runPublished(rest, flags);
  if (command === "consolidate") return runConsolidate(flags);
  if (command === "assemble") return runAssemble(flags);
  if (command === "reconcile") return runReconcile(flags);

  usage(command ? `unknown command: ${command}` : "no command given");
}

main();