diff --git a/agents/trend-spotter.md b/agents/trend-spotter.md index 05b7e16..6b48756 100644 --- a/agents/trend-spotter.md +++ b/agents/trend-spotter.md @@ -283,22 +283,26 @@ this agent. For every trend that cleared the relevance filter (Step 2) — not only the ones that make the final digest — fold it into the persistent trend store, so the next session reasons over it -instead of re-discovering it. The store dedupes on normalized title+URL and unions topics, so -re-capturing an existing trend is safe (it just enriches the tags): +instead of re-discovering it. Build ONE raw-item batch (the same trends you just scored) and pipe +it through `capture`: it normalizes each item, dedupes on normalized title+URL, unions topics on +re-capture (so re-capturing an existing trend just enriches the tags), and persists the source's +`publishedAt` for later freshness ranking — one call, not one per trend: ```bash cd "${CLAUDE_PLUGIN_ROOT}/scripts/trends" && \ - node --import tsx src/cli.ts add \ - --title "" \ - --url "" \ - --topics "" \ - --source "" \ - --summary "" + echo '[ + {"source":"","title":"","url":"", + "topics":["",""],"publishedAt":"", + "summary":""} + ]' | node --import tsx src/cli.ts capture ``` -`--source` is the tool you actually fetched with (**Research Routing**). Skip this step silently -if the store has no deps installed (an adopter without the trends store) — the digest still -compiles, just without persistence. +`source` is the tool you actually fetched with (**Research Routing**); `publishedAt` is the +source's own publish date — omit the key when unknown (the store's `capturedAt` is set +automatically and stays distinct from it). One `capture` call folds the whole batch and reports +`{added, merged, duplicates, errors}`; content-invalid items land in `errors[]`, never failing the +run. Skip this step silently if the store has no deps installed (an adopter without the trends +store) — the digest still compiles, just without persistence. **Step 5: Compile digest** diff --git a/scripts/test-runner.sh b/scripts/test-runner.sh index 9a8bc56..9abdab3 100755 --- a/scripts/test-runner.sh +++ b/scripts/test-runner.sh @@ -40,8 +40,10 @@ # reconcileRecentPosts by literal name, with a non-vacuity self-test) in Section 16f; # the trends-scorer wiring guard (RE-R1: scripts/trends/src/score.ts encodes both mode # weight-sets AND agents/trend-spotter.md references the scorer CLI 'src/cli.ts score', -# with a non-vacuity self-test) in Section 16g; the assertion-count anti-erosion floor -# (SC6) in Section 18. All are live below (Sections 8–18). +# with a non-vacuity self-test) in Section 16g; the trends-capture wiring guard (RE-R2a: +# scripts/trends/src/cli.ts dispatches `capture` AND agents/trend-spotter.md references the +# capture CLI 'src/cli.ts capture', with a non-vacuity self-test) in Section 16h; the +# assertion-count anti-erosion floor (SC6) in Section 18. All are live below (Sections 8–18). # # Usage: bash scripts/test-runner.sh # bash 3.2-safe: plain arrays only, no `declare -A`, no `mapfile`/`readarray`. @@ -692,7 +694,7 @@ if [ -x "$TR_DIR/node_modules/.bin/tsx" ]; then TR_OUT=$( set +e; (cd "$TR_DIR" && npm test) 2>&1; echo "TR_EXIT:$?" ) TR_EXIT=$(echo "$TR_OUT" | grep -oE 'TR_EXIT:[0-9]+' | grep -oE '[0-9]+' | head -1) TR_TESTS=$(echo "$TR_OUT" | grep -oE 'tests [0-9]+' | grep -oE '[0-9]+' | tail -1) - TRENDS_TESTS_FLOOR=62 # store 24 + RE-R1: item 18 + score 16 + cli 4 (item-schema + triage-scorer) + TRENDS_TESTS_FLOOR=79 # store 24 + RE-R1: item 18 + score 16 + cli 4 + RE-R2a: store +9 + item +4 + cli +4 (capture bridge + publishedAt) if [ "$TR_EXIT" = "0" ] && [ -n "$TR_TESTS" ] && [ "$TR_TESTS" -ge "$TRENDS_TESTS_FLOOR" ]; then pass "trends-store suite green: $TR_TESTS tests pass (floor $TRENDS_TESTS_FLOOR)" else @@ -1069,6 +1071,50 @@ fi echo "" +# --- Section 16h: Trends Capture Wiring (research-engine RE-R2a) --- +echo "--- Trends Capture Wiring ---" + +# RE-R2a closes the capture loop: the item->store bridge (itemToInput) + a `capture` CLI that +# normalizes a raw batch from stdin and folds it into the store (persisting publishedAt). Two +# literals must hold, grepped EXACT (grep -F), deps-absent-safe (pure grep, no tsx): +# (1) cli.ts dispatches the `capture` subcommand (the handler exists), by the literal +# 'command === "capture"' (the capture path is real, not merely documented); +# (2) agents/trend-spotter.md re-points Step 4.5 to the capture CLI by the literal +# 'src/cli.ts capture' (one normalizing batch call, replacing the N x `add` block). +# Non-vacuity self-test mirrors Sections 16c-16g: the wiring predicate must accept a probe +# carrying the capture-pointer literal and reject one without it. Labelled 16h but placed after +# Section 17 / before Section 18 (anti-erosion must run last so it sees every prior check). +# UNCONDITIONAL (no tsx) -> counts toward ASSERT_BASELINE_FLOOR. +CAPTURE_HANDLER_LIT='command === "capture"' +CAPTURE_WIRE_LIT='src/cli.ts capture' + +H16_SELFTEST_OK=1 +if ! echo 'pipe the raw batch to src/cli.ts capture for the store fold' | grep -qF "$CAPTURE_WIRE_LIT"; then + H16_SELFTEST_OK=0; echo " non-vacuity FAIL: a wired capture-pointer probe was not detected" +fi +if echo 'the agent folds each trend into the store itself' | grep -qF "$CAPTURE_WIRE_LIT"; then + H16_SELFTEST_OK=0; echo " false-positive FAIL: an unwired probe matched the capture pointer" +fi +if [ "$H16_SELFTEST_OK" -eq 1 ]; then + pass "trends-capture self-test: capture-pointer predicate detects wiring, rejects the under-wired form" +else + fail "trends-capture self-test failed — the capture-wiring lint is vacuous or over-eager" +fi + +if grep -qF "$CAPTURE_HANDLER_LIT" scripts/trends/src/cli.ts; then + pass "cli.ts dispatches the capture subcommand ('$CAPTURE_HANDLER_LIT')" +else + fail "cli.ts has no capture handler — add a '$CAPTURE_HANDLER_LIT' branch (RE-R2a capture loop)" +fi + +if grep -qF "$CAPTURE_WIRE_LIT" agents/trend-spotter.md; then + pass "trend-spotter.md references the capture CLI ('$CAPTURE_WIRE_LIT') as the Step 4.5 store-fold owner" +else + fail "trend-spotter.md does not reference the capture CLI — re-point Step 4.5 to a '$CAPTURE_WIRE_LIT' batch call (RE-R2a wiring)" +fi + +echo "" + # --- Section 18: Assertion-Count Anti-Erosion (SC6) --- # The lint self-modifies its own checks, so a green run could mask a silently dropped # assertion. Pin the total pass()+fail() invocations as a monotonic floor; the count @@ -1080,12 +1126,14 @@ echo "" # checks (ops-reader self-test + strategy-advisor ops-wiring grep) = 82; +2 for SB-S3e's # two UNCONDITIONAL Section-16f checks (reconcile self-test + brain-CLI reconcile grep) = 84; # +3 for RE-R1's three UNCONDITIONAL Section-16g checks (trends-scorer self-test + score.ts -# both-modes weight-set grep + trend-spotter scorer-pointer grep) = 87. +# both-modes weight-set grep + trend-spotter scorer-pointer grep) = 87; +3 for RE-R2a's three +# UNCONDITIONAL Section-16h checks (trends-capture self-test + cli.ts capture-handler grep + +# trend-spotter capture-pointer grep) = 90. # NB: the floor tracks the deps-absent MINIMUM (conditional TS suites warn-skip and drop # the count), so it is bumped only by UNCONDITIONAL new checks — NOT pinned to the # deps-present TOTAL_CHECKS (that would zero the warn-skip margin and false-fail a fresh # clone). Runs last so TOTAL_CHECKS sees every prior check. -ASSERT_BASELINE_FLOOR=87 +ASSERT_BASELINE_FLOOR=90 TOTAL_CHECKS=$((PASS + FAIL)) if [ "$TOTAL_CHECKS" -ge "$ASSERT_BASELINE_FLOOR" ]; then pass "assertion-count anti-erosion: $TOTAL_CHECKS checks >= baseline floor $ASSERT_BASELINE_FLOOR" diff --git a/scripts/trends/README.md b/scripts/trends/README.md index d1c63d4..6ba33d5 100644 --- a/scripts/trends/README.md +++ b/scripts/trends/README.md @@ -35,7 +35,8 @@ interface TrendRecord { title: string; // headline, verbatim url: string; // source URL, verbatim source: string; // "tavily" | "websearch" | "manual" | - capturedAt: string; // ISO-8601 date + capturedAt: string; // ISO-8601 date — when WE captured it + publishedAt?: string;// optional source publish date (ISO-8601); distinct from capturedAt, first-sight, never back-filled topics: string[]; // query tags; unioned across re-captures summary?: string; // optional, verbatim } @@ -47,7 +48,16 @@ slice without breaking the shape. ## CLI ```bash -# Capture a freshly-polled trend (dedupes on title+url; unions topics on re-capture) +# Capture freshly-polled trends — the NORMALIZING BATCH path (the research agent's path): +# raw items on stdin → validate+normalize each → dedupe on title+url → union topics on +# re-capture → persist the source's publishedAt. Content-invalid items are reported in the +# summary errors[], never fail the run; the summary is {added, duplicates, merged, errors}. +echo '[{"source":"tavily","title":"Agentic workflows hit production", + "url":"https://example.com/agentic","topics":["agents","engineering"], + "publishedAt":"2026-06-20","summary":"Teams ship multi-step agents past the demo stage."}]' \ + | node --import tsx src/cli.ts capture [--store ] [--json] + +# Add a SINGLE trend MANUALLY — raw flags, no normalization, publish-date-free: node --import tsx src/cli.ts add \ --title "Agentic workflows hit production" \ --url "https://example.com/agentic" \ @@ -61,7 +71,8 @@ node --import tsx src/cli.ts query --topics "agents,engineering" [--json] node --import tsx src/cli.ts list [--since 2026-06-01] [--limit 10] [--json] ``` -Re-running `add` with the same title+url never appends a duplicate. +Both `capture` and `add` dedupe on normalized title+url — re-capturing the same trend +never appends a duplicate, it only unions any new topics in. ## Tests diff --git a/scripts/trends/src/cli.ts b/scripts/trends/src/cli.ts index b635612..c368926 100644 --- a/scripts/trends/src/cli.ts +++ b/scripts/trends/src/cli.ts @@ -9,15 +9,20 @@ * node --import tsx src/cli.ts status [--store ] [--json] * echo '' | node --import tsx src/cli.ts normalize * echo '' | node --import tsx src/cli.ts score [--mode kortform|long-form] [--threshold N] + * echo '' | node --import tsx src/cli.ts capture [--store ] [--json] * - * The capture agent (research-engine) calls `add` to fold a freshly-polled trend - * into the store, and `query`/`list` to reason over accumulated history. The + * The capture agent (research-engine) folds freshly-polled trends into the store via + * `capture` (the normalizing batch path: stdin → normalizeItem(s) → itemToInput → + * addTrend), and reasons over accumulated history via `query`/`list`. `add` is the + * MANUAL single-trend path (raw flags, no normalization, publish-date-free). The * polling + relevance-scoring itself lives upstream; this is the deterministic store. * - * `normalize` + `score` (RE-R1) are the deterministic research-engine seam: both read - * their JSON PAYLOAD FROM STDIN (so they do not overload `--json`, which stays an - * output toggle) and print JSON to stdout. `normalize` validates raw items into the - * canonical envelope; `score` triages scored candidates (composite/band/threshold). + * `normalize` + `score` (RE-R1) and `capture` (RE-R2a) are the deterministic + * research-engine seam: all read their JSON PAYLOAD FROM STDIN (so they do not overload + * `--json`, which stays an output toggle). `normalize` validates raw items into the + * canonical envelope; `score` triages scored candidates (composite/band/threshold); + * `capture` normalizes + folds each valid item into the store (persisting `publishedAt`), + * reporting content-invalid items in the summary `errors[]`, never via the exit code. * * Exit code: 0 on success, 2 on usage error (incl. unparseable stdin / bad flag). */ @@ -33,7 +38,7 @@ import { queryByTopic, saveStore, } from "./store.js"; -import { normalizeItem, normalizeItems } from "./item.js"; +import { normalizeItem, normalizeItems, itemToInput } from "./item.js"; import { triage } from "./score.js"; import type { ScoreMode } from "./score.js"; @@ -72,7 +77,8 @@ function usage(msg: string): never { " list [--since ] [--limit ] [--store ] [--json]\n" + " status [--store ] [--json]\n" + " normalize < raw-item-or-batch.json\n" + - " score [--mode kortform|long-form] [--threshold N] < scored-candidates.json", + " score [--mode kortform|long-form] [--threshold N] < scored-candidates.json\n" + + " capture [--store ] [--json] < raw-item-or-batch.json", ); process.exit(2); } @@ -228,6 +234,34 @@ function main(): void { return; } + if (command === "capture") { + const payload = readStdinJson(); // exits 2 on empty/unparseable stdin + const raw = Array.isArray(payload) ? payload : [payload]; + const { items, errors } = normalizeItems(raw); + const store = loadStore(storePath); + // Tally derived from AddResult {added, merged} (no `duplicates` field): a fold is + // `added` (new), else `merged` (existing gained topics), else a plain `duplicate`. + let added = 0; + let merged = 0; + let duplicates = 0; + for (const item of items) { + const res = addTrend(store, itemToInput(item, today())); + if (res.added) added++; + else if (res.merged) merged++; + else duplicates++; + } + saveStore(storePath, store); + if (asJson) { + console.log(JSON.stringify({ added, duplicates, merged, errors }, null, 2)); + return; + } + console.log( + `Captured into ${storePath}: ${added} added, ${merged} merged, ` + + `${duplicates} duplicate, ${errors.length} invalid (${store.trends.length} total)`, + ); + return; + } + usage(command ? `unknown command: ${command}` : "no command given"); } diff --git a/scripts/trends/src/item.ts b/scripts/trends/src/item.ts index 0fafb45..30119c7 100644 --- a/scripts/trends/src/item.ts +++ b/scripts/trends/src/item.ts @@ -17,6 +17,7 @@ */ import { normalizeField } from "./store.js"; +import type { TrendInput } from "./store.js"; export interface TrendItem { /** Capture origin: a research-MCP name ("tavily"), "websearch", or "manual". Stored VERBATIM. */ @@ -117,6 +118,26 @@ export function normalizeItem(raw: unknown): NormalizeResult { return { ok: true, item }; } +/** + * Map a validated envelope to a store input (the item→store bridge RE-R1 deferred). + * Pure: injects `capturedAt` (the store's "when WE saw it", supplied by the caller — + * never derived here) and carries the rest verbatim. Does NOT re-validate (the item is + * already validated by normalizeItem) and does NOT derive an `id` (the store owns id via + * addTrend→trendId). `publishedAt`/`summary` are carried only when present (key omitted + * otherwise), mirroring the store's conditional-spread idiom. + */ +export function itemToInput(item: TrendItem, capturedAt: string): TrendInput { + return { + source: item.source, + title: item.title, + url: item.url, + capturedAt, + topics: [...item.topics], + ...(item.publishedAt !== undefined ? { publishedAt: item.publishedAt } : {}), + ...(item.summary !== undefined ? { summary: item.summary } : {}), + }; +} + /** Partition a raw batch into normalized items + per-index errors (never throws). */ export function normalizeItems(raw: unknown[]): { items: TrendItem[]; errors: ItemError[] } { const items: TrendItem[] = []; diff --git a/scripts/trends/src/store.ts b/scripts/trends/src/store.ts index 52d359f..4c4faf5 100644 --- a/scripts/trends/src/store.ts +++ b/scripts/trends/src/store.ts @@ -28,6 +28,8 @@ export interface TrendInput { url: string; source: string; capturedAt: string; + /** The source's own publish date (ISO-8601), if known. Distinct from capturedAt; persisted first-sight, never back-filled. */ + publishedAt?: string; topics: string[]; summary?: string; } @@ -74,8 +76,15 @@ export function emptyStore(): TrendStore { export function loadStore(path: string): TrendStore { if (!existsSync(path)) return emptyStore(); const parsed = JSON.parse(readFileSync(path, "utf8")) as Partial; + // Forward migrate-on-load: stamp to the current version, never downgrade. v1→v2 is + // purely additive-optional (an old record is already a valid v2 record that simply + // lacks the optional publishedAt), so the migration is the version stamp alone — + // records pass through untouched (lossless + idempotent for any well-formed store). + // A string / NaN / absent version coerces to the current version (never crashes); the + // non-array `trends` coercion below is unchanged and out of the losslessness claim. + const onDisk = typeof parsed.schemaVersion === "number" ? parsed.schemaVersion : SCHEMA_VERSION; return { - schemaVersion: parsed.schemaVersion ?? SCHEMA_VERSION, + schemaVersion: Math.max(onDisk, SCHEMA_VERSION), trends: Array.isArray(parsed.trends) ? parsed.trends : [], }; } @@ -122,6 +131,7 @@ export function addTrend(store: TrendStore, input: TrendInput): AddResult { url: input.url, source: input.source, capturedAt: input.capturedAt, + ...(input.publishedAt !== undefined ? { publishedAt: input.publishedAt } : {}), topics: [...input.topics], ...(input.summary !== undefined ? { summary: input.summary } : {}), }; diff --git a/scripts/trends/src/types.ts b/scripts/trends/src/types.ts index 3427066..430acb5 100644 --- a/scripts/trends/src/types.ts +++ b/scripts/trends/src/types.ts @@ -34,6 +34,13 @@ export interface TrendRecord { source: string; /** ISO-8601 date the trend was captured. Supplied by the caller (CLI edge). */ capturedAt: string; + /** + * The SOURCE's own publish date (ISO-8601), if the captured item carried one — + * distinct from `capturedAt` (when WE saw it): this is when the source published. + * First-sight provenance: kept, never overwritten on re-capture. Forward-compat + * for B4 freshness ranking. Absent when the source gave no date (key omitted). + */ + publishedAt?: string; /** Topic tags for query-by-topic. Unioned across re-captures of the same trend. */ topics: string[]; /** Optional short summary of the trend, stored VERBATIM. */ @@ -52,4 +59,4 @@ export interface TrendQueryHit { topicOverlap: number; } -export const SCHEMA_VERSION = 1; +export const SCHEMA_VERSION = 2; diff --git a/scripts/trends/tests/cli.test.ts b/scripts/trends/tests/cli.test.ts index 58ac8f2..1b73278 100644 --- a/scripts/trends/tests/cli.test.ts +++ b/scripts/trends/tests/cli.test.ts @@ -2,6 +2,9 @@ import { describe, test } from "node:test"; import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; +import { mkdtempSync, rmSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; // Resolve the package root (scripts/trends) so the subprocess `src/cli.ts` path + the // `tsx` loader resolve regardless of the runner's cwd. @@ -67,4 +70,92 @@ describe("trends CLI — normalize/score subcommands (RE-R1 / Step 4)", () => { assert.equal(status, 2); }); }); + + describe("capture (stdin raw item|batch -> folds into the store) (RE-R2a / Step 4)", () => { + const tmpStore = () => join(mkdtempSync(join(tmpdir(), "trends-capture-")), "trends.json"); + + test("happy path: a valid item piped in -> folded into the store, added:1, publishedAt persisted", () => { + const store = tmpStore(); + try { + const batch = JSON.stringify([ + { + source: "tavily", + title: "Captured", + url: "https://example.com/c", + topics: ["ai"], + publishedAt: "2026-06-20", + }, + ]); + const { status, stdout } = run(["capture", "--store", store, "--json"], batch); + assert.equal(status, 0); + const summary = JSON.parse(stdout); + assert.equal(summary.added, 1); + assert.equal(summary.errors.length, 0); + assert.equal( + summary.added + summary.merged + summary.duplicates + summary.errors.length, + 1, + "tally must sum to the input size", + ); + const persisted = JSON.parse(readFileSync(store, "utf8")); + assert.equal(persisted.schemaVersion, 2); + assert.equal(persisted.trends.length, 1); + assert.equal(persisted.trends[0].publishedAt, "2026-06-20"); + assert.match(persisted.trends[0].capturedAt, /^\d{4}-\d{2}-\d{2}$/); + assert.notEqual( + persisted.trends[0].capturedAt, + persisted.trends[0].publishedAt, + "capturedAt (when WE saw it) must be distinct from publishedAt (source date)", + ); + } finally { + rmSync(join(store, ".."), { recursive: true, force: true }); + } + }); + + test("a batch with one content-invalid item -> valid added, invalid in errors[], exit 0", () => { + const store = tmpStore(); + try { + const batch = JSON.stringify([ + { source: "tavily", title: "Valid", url: "https://example.com/v", topics: ["x"] }, + { title: "no source or url" }, + ]); + const { status, stdout } = run(["capture", "--store", store, "--json"], batch); + assert.equal(status, 0); + const summary = JSON.parse(stdout); + assert.equal(summary.added, 1); + assert.equal(summary.errors.length, 1); + assert.equal( + summary.added + summary.merged + summary.duplicates + summary.errors.length, + 2, + ); + } finally { + rmSync(join(store, ".."), { recursive: true, force: true }); + } + }); + + test("re-capturing the same trend with a new topic -> merged:1, tally still sums", () => { + const store = tmpStore(); + try { + const item = (topics: string[]) => + JSON.stringify([{ source: "tavily", title: "Dup", url: "https://example.com/d", topics }]); + run(["capture", "--store", store, "--json"], item(["a"])); + const { status, stdout } = run(["capture", "--store", store, "--json"], item(["a", "b"])); + assert.equal(status, 0); + const summary = JSON.parse(stdout); + assert.equal(summary.added, 0); + assert.equal(summary.merged, 1); + assert.equal(summary.duplicates, 0); + assert.equal( + summary.added + summary.merged + summary.duplicates + summary.errors.length, + 1, + ); + } finally { + rmSync(join(store, ".."), { recursive: true, force: true }); + } + }); + + test("bad invocation: empty stdin -> exit 2", () => { + const { status } = run(["capture"], ""); + assert.equal(status, 2); + }); + }); }); diff --git a/scripts/trends/tests/item.test.ts b/scripts/trends/tests/item.test.ts index 8996f82..a059913 100644 --- a/scripts/trends/tests/item.test.ts +++ b/scripts/trends/tests/item.test.ts @@ -1,7 +1,7 @@ import { describe, test } from "node:test"; import assert from "node:assert/strict"; -import { normalizeItem, normalizeItems } from "../src/item.js"; +import { normalizeItem, normalizeItems, itemToInput } from "../src/item.js"; import { normalizeField } from "../src/store.js"; describe("trends item normalizer (RE-R1 / B1)", () => { @@ -179,4 +179,49 @@ describe("trends item normalizer (RE-R1 / B1)", () => { assert.deepEqual(errors, []); }); }); + + describe("itemToInput — bridge to the store input (RE-R2a / B-bridge)", () => { + // Build a realistic envelope via the normalizer (the actual upstream path). + const mkItem = (extra: Record = {}) => { + const r = normalizeItem({ + source: "tavily", + title: "OpenAI ships a reasoning model", + url: "https://example.com/Article", + topics: ["ai", "reasoning"], + summary: "A short summary.", + ...extra, + }); + assert.equal(r.ok, true); + if (!r.ok) throw new Error("fixture item failed to normalize"); + return r.item; + }; + + test("injects capturedAt and carries source/title/url/topics/summary/publishedAt verbatim", () => { + const input = itemToInput(mkItem({ publishedAt: "2026-06-20" }), "2026-06-24"); + assert.equal(input.capturedAt, "2026-06-24"); + assert.equal(input.source, "tavily"); + assert.equal(input.title, "OpenAI ships a reasoning model"); + assert.equal(input.url, "https://example.com/Article"); + assert.deepEqual(input.topics, ["ai", "reasoning"]); + assert.equal(input.summary, "A short summary."); + assert.equal(input.publishedAt, "2026-06-20"); + }); + + test("derives NO id (the store owns id via addTrend->trendId)", () => { + const input = itemToInput(mkItem(), "2026-06-24") as Record; + assert.equal("id" in input, false); + }); + + test("an item without publishedAt -> input omits the key (not undefined-valued)", () => { + const input = itemToInput(mkItem(), "2026-06-24"); // mkItem carries no publishedAt + assert.equal("publishedAt" in input, false); + }); + + test("field-confusion guard: capturedAt (injected) is distinct from publishedAt (carried)", () => { + const input = itemToInput(mkItem({ publishedAt: "2026-06-20" }), "2026-06-24"); + assert.notEqual(input.capturedAt, input.publishedAt); + assert.equal(input.publishedAt, "2026-06-20"); + assert.equal(input.capturedAt, "2026-06-24"); + }); + }); }); diff --git a/scripts/trends/tests/store.test.ts b/scripts/trends/tests/store.test.ts index a149329..e5280f8 100644 --- a/scripts/trends/tests/store.test.ts +++ b/scripts/trends/tests/store.test.ts @@ -1,6 +1,6 @@ import { describe, test } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, rmSync, existsSync } from "node:fs"; +import { mkdtempSync, rmSync, existsSync, writeFileSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; @@ -206,6 +206,75 @@ describe("trends store", () => { assert.equal(res2.added, false); assert.equal(res2.merged, false, "no new tags → merged:false"); }); + + // ── RE-R2a: publishedAt persistence ── + test("RED: persists publishedAt on a new record when present", () => { + const res = addTrend(emptyStore(), { + title: "Dated trend", + url: "https://example.com/d", + source: "tavily", + capturedAt: "2026-06-24", + topics: ["ai"], + publishedAt: "2026-06-20", + }); + assert.equal(res.store.trends[0].publishedAt, "2026-06-20"); + }); + + test("regression guard: omits publishedAt when absent (no undefined-valued key)", () => { + const res = addTrend(emptyStore(), { + title: "Undated trend", + url: "https://example.com/u", + source: "tavily", + capturedAt: "2026-06-24", + topics: ["ai"], + }); + assert.equal("publishedAt" in res.store.trends[0], false); + }); + + test("RED: re-capture keeps the first sighting's publishedAt (no overwrite), unions topics", () => { + let store = emptyStore(); + store = addTrend(store, { + title: "Same dated trend", + url: "https://example.com/sd", + source: "tavily", + capturedAt: "2026-06-01", + topics: ["a"], + publishedAt: "2026-05-30", + }).store; + const res2 = addTrend(store, { + title: "Same dated trend", + url: "https://example.com/sd", + source: "gemini", + capturedAt: "2026-06-20", + topics: ["a", "b"], + publishedAt: "2026-06-15", + }); + assert.equal(res2.added, false); + assert.equal(res2.merged, true); + assert.equal(res2.store.trends[0].publishedAt, "2026-05-30", "first-sight publishedAt kept"); + assert.deepEqual([...res2.store.trends[0].topics].sort(), ["a", "b"]); + }); + + test("regression guard: no back-fill — re-capture carrying publishedAt onto a record that lacked one does NOT add it", () => { + let store = emptyStore(); + store = addTrend(store, { + title: "Undated first sight", + url: "https://example.com/uf", + source: "tavily", + capturedAt: "2026-06-01", + topics: ["a"], + }).store; // no publishedAt on first sight + const res2 = addTrend(store, { + title: "Undated first sight", + url: "https://example.com/uf", + source: "tavily", + capturedAt: "2026-06-20", + topics: ["a", "b"], + publishedAt: "2026-06-15", + }); + assert.equal(res2.merged, true, "topic union still reported"); + assert.equal("publishedAt" in res2.store.trends[0], false, "no back-fill of first-sight provenance"); + }); }); describe("queryByTopic", () => { @@ -330,4 +399,98 @@ describe("trends store", () => { assert.equal(newestCaptureDate(store), "2026-06-01"); }); }); + + describe("schema migration (RE-R2a / publishedAt v1→v2)", () => { + const withFixture = (contents: string, fn: (path: string) => void) => { + const dir = tmp(); + const path = join(dir, "trends.json"); + try { + writeFileSync(path, contents, "utf8"); + fn(path); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }; + + // ── genuinely RED: old loadStore returns the on-disk/garbage version unchanged ── + test("RED: a v1 store loads stamped as v2, records intact, no publishedAt invented", () => { + const v1 = JSON.stringify({ + schemaVersion: 1, + trends: [ + { + id: "abc123", + title: "Old trend", + url: "https://example.com/o", + source: "tavily", + capturedAt: "2026-05-01", + topics: ["ai"], + }, + ], + }); + withFixture(v1, (path) => { + const s = loadStore(path); + assert.equal(s.schemaVersion, 2, "v1 store must migrate to v2"); + assert.equal(s.trends.length, 1); + assert.equal(s.trends[0].title, "Old trend"); + assert.equal(s.trends[0].capturedAt, "2026-05-01"); + assert.deepEqual(s.trends[0].topics, ["ai"]); + assert.equal("publishedAt" in s.trends[0], false, "migration must not invent publishedAt"); + }); + }); + + test("RED: round-trip loadStore→saveStore writes schemaVersion:2 to disk", () => { + withFixture(JSON.stringify({ schemaVersion: 1, trends: [] }), (path) => { + saveStore(path, loadStore(path)); + const onDisk = JSON.parse(readFileSync(path, "utf8")); + assert.equal(onDisk.schemaVersion, 2); + }); + }); + + test("RED: a non-numeric schemaVersion is coerced to v2 (old code passes the string through)", () => { + withFixture(JSON.stringify({ schemaVersion: "weird", trends: [] }), (path) => { + assert.equal(loadStore(path).schemaVersion, 2); + }); + }); + + // ── GREEN-only regression guards: old code already returns the current version ── + test("regression guard: a v2 store loads as v2, idempotent (records + publishedAt intact)", () => { + const v2 = JSON.stringify({ + schemaVersion: 2, + trends: [ + { + id: "x", + title: "T", + url: "https://example.com/t", + source: "tavily", + capturedAt: "2026-06-01", + topics: ["ai"], + publishedAt: "2026-05-30", + }, + ], + }); + withFixture(v2, (path) => { + const s = loadStore(path); + assert.equal(s.schemaVersion, 2); + assert.equal(s.trends[0].publishedAt, "2026-05-30"); + }); + }); + + test("regression guard: a store with no schemaVersion is stamped to the current version", () => { + const noVer = JSON.stringify({ + trends: [ + { + id: "x", + title: "T", + url: "https://example.com/t", + source: "tavily", + capturedAt: "2026-06-01", + topics: ["ai"], + }, + ], + }); + withFixture(noVer, (path) => { + assert.equal(loadStore(path).schemaVersion, SCHEMA_VERSION); + }); + }); + }); });